revision

README.md

@@ -1,13 +1,3 @@
----
-title: Zero2Story
-emoji: 📖
-sdk: gradio
-sdk_version: 3.42.0
-app_file: app.py
-pinned: false
-license: apache-2.0
----
-
 # Zero2Story
 
 ![](assets/overview.png)
@@ -35,7 +25,7 @@ This project heavily depends on [PaLM API](https://developers.generativeai.googl
 Make sure you have installed all of the following prerequisites on your development machine:
 * CUDA Toolkit 11.8 with cuDNN 8 - [Download & Install CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) It is highly recommended to run on a GPU. If you run it in a CPU environment, it will be very slow.
 * Poetry - [Download & Install Poetry](https://python-poetry.org/docs/#installation) It is the python packaging and dependency manager.
-* SQLite3 v3.37.2 or higher - It is required to be installed due to dependencies.
+* SQLite3 v3.35.0 or higher - It is required to be installed due to dependencies.
     - Ubuntu 22.04 and later
     ```shell
     $ sudo apt install libc6 sqlite3 libsqlite3
@@ -53,10 +43,21 @@ Make sure you have installed all of the following prerequisites on your developm
     ```shell
     $ sudo apt install ffmpeg
 
-## Run
+## Installation
 
+Before running the application for the first time, install the required dependencies:
 ```shell
 $ poetry install
+```
+
+If dependencies change or need updates in the future, you can use:
+```shell
+$ poetry update
+```
+
+## Run
+
+```shell
 $ poetry run python app.py
 ```
 
@@ -73,4 +74,4 @@ $ poetry run python app.py
 
 ### VAEs
 - For character image generation: [CIVIT.AI Model 23906](https://civitai.com/models/23906)
-- For background image generation: [CIVIT.AI Model 65728](https://civitai.com/models/65728)
+- For background image generation: [CIVIT.AI Model 65728](https://civitai.com/models/65728)

app.py

@@ -10,16 +10,14 @@ from constants.init_values import (
 from constants import desc
 
 from interfaces import (
-    ui, chat_ui, story_gen_ui, view_change_ui
+    ui, chat_ui, story_gen_ui, view_change_ui, export_ui
 )
 from modules.palmchat import GradioPaLMChatPPManager
 
 with gr.Blocks(css=STYLE) as demo:
-	chat_mode = gr.State("plot_chat")
-
+	chat_mode = gr.State("setting_chat")
 	chat_state = gr.State({
-		"ppmanager_type": GradioPaLMChatPPManager(),
-		"plot_chat": GradioPaLMChatPPManager(),
+		"setting_chat": GradioPaLMChatPPManager(),
 		"story_chat": GradioPaLMChatPPManager(),
 		"export_chat": GradioPaLMChatPPManager(),
 	})
@@ -31,6 +29,11 @@ with gr.Blocks(css=STYLE) as demo:
 	gallery_images2 = gr.State(default_character_images)
 	gallery_images3 = gr.State(default_character_images)
 	gallery_images4 = gr.State(default_character_images)
+ 
+	selected_main_char_image1 = gr.State(default_character_images[0])
+	selected_side_char_image1 = gr.State(default_character_images[0])
+	selected_side_char_image2 = gr.State(default_character_images[0])
+	selected_side_char_image3 = gr.State(default_character_images[0])
 
 	with gr.Column(visible=True) as pre_phase:
 		gr.Markdown("# 📖 Zero2Story", elem_classes=["markdown-center"])
@@ -242,16 +245,6 @@ with gr.Blocks(css=STYLE) as demo:
 				"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Integer interdum eleifend tincidunt. Vivamus dapibus, massa ut imperdiet condimentum, quam ipsum vehicula eros, a accumsan nisl metus at nisl. Nullam tortor nibh, vehicula sed tellus at, accumsan efficitur enim. Sed mollis purus vitae nisl ornare volutpat. In vitae tortor nec neque sagittis vehicula. In vestibulum velit eu lorem pulvinar dignissim. Donec eu sapien et sapien cursus pretium elementum eu urna. Proin lacinia ipsum maximus, commodo dui tempus, convallis tortor. Nulla sodales mi libero, nec eleifend eros interdum quis. Pellentesque nulla lectus, scelerisque et consequat vitae, blandit at ante. Sed nec …….",
 				lines=12,
 				elem_classes=["no-label", "small-big-textarea"]
-		)		
-
-		action_types = gr.Radio(
-			choices=[
-				"continue current phase", "move to the next phase"
-			],
-			value="continue current phase",
-			interactive=True,
-			elem_classes=["no-label-radio"],
-			visible=False,
 		)
 
 		with gr.Accordion("regeneration controls", open=False):
@@ -259,7 +252,7 @@ with gr.Blocks(css=STYLE) as demo:
 				regen_actions_btn = gr.Button("Re-suggest actions", interactive=True, elem_classes=["control-button-green"])
 				regen_story_btn = gr.Button("Re-suggest story and actions", interactive=True, elem_classes=["control-button-green"])
 	
-			custom_prompt_txt = gr.Textbox(placeholder="Re-suggest story and actions based on your own custom request", elem_classes=["no-label", "small-big-textarea"])
+			custom_prompt_txt = gr.Textbox(placeholder="Re-suggest story and actions based on your own custom request", interactive=True, elem_classes=["no-label", "small-big-textarea"])
 
 		with gr.Row():
 			action_btn1 = gr.Button("Action Choice 1", interactive=False, elem_classes=["control-button-green"])
@@ -273,9 +266,23 @@ with gr.Blocks(css=STYLE) as demo:
 			story_writing_done_btn = gr.Button("export your story →", elem_classes=["wrap", "control-button"], scale=2)
 
 	with gr.Column(visible=False) as export_phase:
-		gr.Markdown("### 📤 Export output")
-		with gr.Accordion("generate chapter titles and each plot", open=False) as export_section:
-			gr.Markdown("hello")
+		gr.Markdown("# 📤 Story writing")
+		gr.Markdown(desc.export_phase_description, elem_classes=["markdown-justify"])
+
+		title_txt = gr.Textbox("Your Own Story", elem_classes=["no-label"])
+		title_gen_btn = gr.Button("gnerate a title", elem_classes=["control-button-green"])
+
+		with gr.Row():
+			back_to_story_writing_btn = gr.Button("← back", elem_classes=["wrap", "control-button"], scale=1)
+			restart_from_export_btn = gr.Button("start over", elem_classes=["wrap", "control-button"], scale=1)
+			export_done_btn = gr.Button("exported story →", elem_classes=["wrap", "control-button"], scale=1)
+
+	with gr.Column(visible=False) as export_view_phase:
+		export_html = gr.HTML()
+
+		with gr.Row():
+			restart_from_export_view_btn = gr.Button("start over", elem_classes=["wrap", "control-button"])
+			export_to_file_btn = gr.Button("Download as ZIP file", elem_classes=["wrap", "control-button"], scale=2)
 
 	with gr.Accordion("💬", open=False, elem_id="chat-section") as chat_section:
 		with gr.Column(scale=1):
@@ -320,6 +327,91 @@ with gr.Blocks(css=STYLE) as demo:
 		outputs=[pre_phase, writing_phase]		
 	)
 
+	story_writing_done_btn.click(
+		view_change_ui.move_to_next_view,
+		inputs=None,
+		outputs=[writing_phase, export_phase]
+	)
+
+	title_gen_btn.click(
+		export_ui.title_gen,
+		inputs=[cursors],
+		outputs=[title_txt]
+	)
+
+	export_done_btn.click(
+		view_change_ui.move_to_next_view,
+		inputs=None,
+		outputs=[export_phase, export_view_phase]
+	).then(
+		export_ui.export,
+		inputs=[
+			title_txt,
+			cursors, 
+			selected_main_char_image1, name_txt1, age_dd1, mbti_dd1, personality_dd1, job_dd1,
+			side_char_enable_ckb1, selected_side_char_image1, name_txt2, age_dd2, mbti_dd2, personality_dd2, job_dd2,
+			side_char_enable_ckb2, selected_side_char_image2, name_txt3, age_dd3, mbti_dd3, personality_dd3, job_dd3,
+			side_char_enable_ckb3, selected_side_char_image3, name_txt4, age_dd4, mbti_dd4, personality_dd4, job_dd4,			
+		],
+		outputs=[
+			export_html
+		]
+	)
+ 
+	back_to_story_writing_btn.click(
+		view_change_ui.back_to_previous_view,
+		inputs=None,
+		outputs=[writing_phase, export_phase]
+	)
+
+	restart_from_export_view_btn.click(
+		view_change_ui.back_to_previous_view,
+		inputs=None,
+		outputs=[pre_phase, export_view_phase]		
+	).then(
+		ui.reset,
+		inputs=None,
+		outputs=[
+			cursors, cur_cursor,
+			chat_state, chat_mode,
+			gallery_images1, gallery_images2, gallery_images3, gallery_images4,
+			selected_main_char_image1, selected_side_char_image1, selected_side_char_image2, selected_side_char_image3,
+			genre_dd, place_dd, mood_dd, 
+			char_gallery1, job_dd1,
+			side_char_enable_ckb1, char_gallery2, job_dd2,
+			side_char_enable_ckb2, char_gallery3, job_dd3,
+			side_char_enable_ckb3, char_gallery4, job_dd4,
+			story_image, story_audio, story_video,
+			story_content, story_progress,
+			custom_prompt_txt, action_btn1, action_btn2, action_btn3, custom_action_txt,
+			title_txt, export_html
+		]
+	)
+
+	restart_from_export_btn.click(
+		view_change_ui.back_to_previous_view,
+		inputs=None,
+		outputs=[pre_phase, export_phase]
+	).then(
+		ui.reset,
+		inputs=None,
+		outputs=[
+			cursors, cur_cursor,
+			chat_state, chat_mode,
+			gallery_images1, gallery_images2, gallery_images3, gallery_images4,
+			selected_main_char_image1, selected_side_char_image1, selected_side_char_image2, selected_side_char_image3,
+			genre_dd, place_dd, mood_dd, 
+			char_gallery1, job_dd1,
+			side_char_enable_ckb1, char_gallery2, job_dd2,
+			side_char_enable_ckb2, char_gallery3, job_dd3,
+			side_char_enable_ckb3, char_gallery4, job_dd4,
+			story_image, story_audio, story_video,
+			story_content, story_progress,
+			custom_prompt_txt, action_btn1, action_btn2, action_btn3, custom_action_txt,
+			title_txt, export_html
+		]
+	)
+
 	character_setup_confirm_btn.click(
 		view_change_ui.move_to_next_view,
 		inputs=None,
@@ -407,25 +499,25 @@ with gr.Blocks(css=STYLE) as demo:
 		ui.gen_character_image,
 		inputs=[
 			gallery_images1, name_txt1, age_dd1, mbti_dd1, personality_dd1, job_dd1, genre_dd, place_dd, mood_dd, creative_dd1],
-		outputs=[char_gallery1, gallery_images1]
+		outputs=[char_gallery1, gallery_images1, selected_main_char_image1]
 	)
 
 	gen_char_btn2.click(
 		ui.gen_character_image,
 		inputs=[gallery_images2, name_txt2, age_dd2, mbti_dd2, personality_dd2, job_dd2, genre_dd, place_dd, mood_dd, creative_dd2],
-		outputs=[char_gallery2, gallery_images2]
+		outputs=[char_gallery2, gallery_images2, selected_side_char_image1]
 	)
 
 	gen_char_btn3.click(
 		ui.gen_character_image,
 		inputs=[gallery_images3, name_txt3, age_dd3, mbti_dd3, personality_dd3, job_dd3, genre_dd, place_dd, mood_dd, creative_dd3],
-		outputs=[char_gallery3, gallery_images3]
+		outputs=[char_gallery3, gallery_images3, selected_side_char_image2]
 	)
 
 	gen_char_btn4.click(
 		ui.gen_character_image,
 		inputs=[gallery_images4, name_txt4, age_dd4, mbti_dd4, personality_dd4, job_dd4, genre_dd, place_dd, mood_dd, creative_dd4],
-		outputs=[char_gallery4, gallery_images4]
+		outputs=[char_gallery4, gallery_images4, selected_side_char_image3]
 	)
 
 	random_name_btn1.click(
@@ -685,4 +777,28 @@ with gr.Blocks(css=STYLE) as demo:
 		outputs=[chat_input_txt, chat_state, chatbot, regen_btn]
 	)
 
-demo.queue().launch()
+	char_gallery1.select(
+		ui.update_selected_char_image,
+		inputs=None,
+		outputs=[selected_main_char_image1]
+	)
+
+	char_gallery2.select(
+		ui.update_selected_char_image,
+		inputs=None,
+		outputs=[selected_side_char_image1]
+	)
+
+	char_gallery3.select(
+		ui.update_selected_char_image,
+		inputs=None,
+		outputs=[selected_side_char_image2]
+	)
+
+	char_gallery4.select(
+		ui.update_selected_char_image,
+		inputs=None,
+		outputs=[selected_side_char_image3]
+	)
+
+demo.queue().launch(share=True)

assets/palm_prompts.toml

@@ -12,7 +12,7 @@ To enhance the quality of your character's description or expression, you might
 - Age and Gender: "1boy", "1man", "1male", "1girl", "1woman", "1female", "teen", "teenage", "twenties", "thirties", "forties", "fifties", "middle-age".
 Do note that this list isn't exhaustive, and you're encouraged to suggest similar terms not included here.
 Exclude words from the suggestion that are redundant or have conflicting meanings.
-Especially, Exclude words that conflict with the meaning of "main_sentence".
+Especially, Exclude words that conflict with the meaning of "primary_sentence".
 Do not output anything other than JSON values.
 Do not provide any additional explanation of the following.
 Only JSON is allowed.
@@ -64,7 +64,7 @@ To enhance the quality of your scene's description or expression, you might cons
 - Visual Enhancements: "masterpiece", "cinematic lighting", "detailed lighting", "soft lighting", "volumetric lighting", "tyndall effect", "warm lighting", "close up", "wide shot", "beautiful perspective", "bokeh".
 Do note that this list isn't exhaustive, and you're encouraged to suggest similar terms not included here.
 Exclude words from the suggestion that are redundant or have conflicting meanings.
-Especially, Exclude words that conflict with the meaning of "main_sentence".
+Especially, Exclude words that conflict with the meaning of "primary_sentence".
 Do not output anything other than JSON values.
 Do not provide any additional explanation of the following.
 Only JSON is allowed.
@@ -74,25 +74,25 @@ Q:
 The genre is "Fantasy", the place is "Enchanted Forest", the mood is "Mystical", the title of the novel is "Whispering Leaves", and the chapter plot revolves around "A hidden glade where elves sing under the moonlight".
 Print out no more than 45 words in syntactically valid JSON format.
 A:
-{{"main_sentence":"a mystical glade in an enchanted forest where elves sing beneath the moonlight","descriptors":["no humans","masterpiece","fantasy","enchanted forest","moonlit glade","mystical atmosphere","singing elves","luminous fireflies","ancient trees","shimmering leaves","whispering winds","hidden secrets","elven magic","masterpiece","soft lighting","silver glow","detailed shadows","enchanted mood","highly detailed","film grain"]}}
+{{"primary_sentence":"a mystical glade in an enchanted forest where elves sing beneath the moonlight","descriptors":["no humans","masterpiece","fantasy","enchanted forest","moonlit glade","mystical atmosphere","singing elves","luminous fireflies","ancient trees","shimmering leaves","whispering winds","hidden secrets","elven magic","masterpiece","soft lighting","silver glow","detailed shadows","enchanted mood","highly detailed","film grain"]}}
 
 Q:
 The genre is "Science Fiction", the place is "Galactic Space Station", the mood is "Tense", the title of the novel is "Stars Unbound", and the chapter plot revolves around "Ambassadors from different galaxies discussing a new treaty".
 Print out no more than 45 words in syntactically valid JSON format.
 A:
-{{"main_sentence":"a tense gathering in a galactic space station where interstellar ambassadors negotiate","descriptors":["no humans","masterpiece","science fiction","galactic space station","star-studded backdrop","advanced technology","diverse aliens","hovering spacecrafts","futuristic architecture","tense discussions","interstellar politics","neon lights","holographic displays","masterpiece","detailed lighting","cinematic mood","highly detailed","film grain"]}}
+{{"primary_sentence":"a tense gathering in a galactic space station where interstellar ambassadors negotiate","descriptors":["no humans","masterpiece","science fiction","galactic space station","star-studded backdrop","advanced technology","diverse aliens","hovering spacecrafts","futuristic architecture","tense discussions","interstellar politics","neon lights","holographic displays","masterpiece","detailed lighting","cinematic mood","highly detailed","film grain"]}}
 
 Q:
 The genre is "Romance", the place is "Beach", the mood is "Heartfelt", the title of the novel is "Waves of Passion", and the chapter plot revolves around "Two lovers reconciling their differences by the shore".
 Print out no more than 45 words in syntactically valid JSON format.
 A:
-{{"main_sentence":"a heartfelt scene on a beach during sunset where two lovers reconcile","descriptors":["no humans","masterpiece","romance","beach","sunset horizon","golden sands","lapping waves","embrace","teary-eyed confessions","seashells","reflective waters","warm hues","silhouette of lovers","soft breeze","beautiful perspective","detailed shadows","emotional atmosphere","highly detailed","film grain"]}}
+{{"primary_sentence":"a heartfelt scene on a beach during sunset where two lovers reconcile","descriptors":["no humans","masterpiece","romance","beach","sunset horizon","golden sands","lapping waves","embrace","teary-eyed confessions","seashells","reflective waters","warm hues","silhouette of lovers","soft breeze","beautiful perspective","detailed shadows","emotional atmosphere","highly detailed","film grain"]}}
 
 Q:
 The genre is "Middle Ages", the place is "Royal Palace", the mood is "Epic Adventure", the title of the novel is "Throne of Fates", and the chapter plot revolves around "A brave knight receiving a quest from the king".
 Print out no more than 45 words in syntactically valid JSON format.
 A:
-{{"main_sentence":"an epic scene in a royal palace where a knight is tasked with a quest by the king","descriptors":["no humans","masterpiece","middle ages","royal palace","castle","grand throne room","golden hour","armored knight","majestic king","tapestries","stone walls","torches","glistening armor","banner flags","medieval atmosphere","heroic demeanor","detailed architecture","golden crowns","highly detailed","film grain"]}}
+{{"primary_sentence":"an epic scene in a royal palace where a knight is tasked with a quest by the king","descriptors":["no humans","masterpiece","middle ages","royal palace","castle","grand throne room","golden hour","armored knight","majestic king","tapestries","stone walls","torches","glistening armor","banner flags","medieval atmosphere","heroic demeanor","detailed architecture","golden crowns","highly detailed","film grain"]}}
 ===
 This is my request.
 Q:
@@ -126,22 +126,22 @@ This is some examples.
 Q:
 The genre is "Fantasy", the place is "Enchanted Forest", the mood is "Mystical", the title of the novel is "Whispering Leaves", and the chapter plot revolves around "A hidden glade where elves sing under the moonlight".
 A:
-{{"main_sentence":"a gentle folk melody filled with whimsical flutes, echoing harps, and distant ethereal vocals, capturing the enchantment of a moonlit forest and the mystique of singing elves"}}
+{{"primary_sentence":"a gentle folk melody filled with whimsical flutes, echoing harps, and distant ethereal vocals, capturing the enchantment of a moonlit forest and the mystique of singing elves"}}
 
 Q:
 The genre is "Science Fiction", the place is "Galactic Space Station", the mood is "Tense", the title of the novel is "Stars Unbound", and the chapter plot revolves around "Ambassadors from different galaxies discussing a new treaty".
 A:
-{{"main_sentence":"an ambient electronic track, with pulsating synths, spacey reverberations, and occasional digital glitches, reflecting the vastness of space and the tension of intergalactic diplomacy"}}
+{{"primary_sentence":"an ambient electronic track, with pulsating synths, spacey reverberations, and occasional digital glitches, reflecting the vastness of space and the tension of intergalactic diplomacy"}}
 
 Q:
 The genre is "Romance", the place is "Beach", the mood is "Heartfelt", the title of the novel is "Waves of Passion", and the chapter plot revolves around "Two lovers reconciling their differences by the shore".
 A:
-{{"main_sentence":"a soft acoustic ballad featuring soulful guitars, delicate percussion, and heartfelt vocals, evoking feelings of love, reconciliation, and the gentle ebb and flow of the ocean waves"}}
+{{"primary_sentence":"a soft acoustic ballad featuring soulful guitars, delicate percussion, and heartfelt vocals, evoking feelings of love, reconciliation, and the gentle ebb and flow of the ocean waves"}}
 
 Q:
 The genre is "Middle Ages", the place is "Royal Palace", the mood is "Epic Adventure", the title of the novel is "Throne of Fates", and the chapter plot revolves around "A brave knight receiving a quest from the king".
 A:
-{{"main_sentence":"a grand orchestral piece, dominated by powerful brass, rhythmic drums, and soaring strings, portraying the valor of knights, the majesty of royalty, and the anticipation of an epic quest"}}
+{{"primary_sentence":"a grand orchestral piece, dominated by powerful brass, rhythmic drums, and soaring strings, portraying the valor of knights, the majesty of royalty, and the anticipation of an epic quest"}}
 ===
 This is my request.
 Q:
@@ -150,5 +150,5 @@ A:
 """
 query = """
 The genre is "{genre}", the place is "{place}", the mood is "{mood}", the title of the novel is "{title}", and the chapter plot revolves around "{chapter_plot}".
-Print out only one main_sentence in syntactically valid JSON format.
+Print out only one primary_sentence in syntactically valid JSON format.
 """

constants/desc.py

@@ -16,4 +16,7 @@ story_generation_phase_description = """
 In this phase, the first few paragraphs are generated solely based on the information from the background and character setup phases. Afterwards, users could choose a direction from the given three options that PaLM API generated. Then, further stories are generated based on users' choice. This cycle of choosing an option and generating further stories are interatively continued until users decides to stop. 
 
 In each story generation, users also could generate background images and music that describe each scene using Stable Diffusion and MusicGen. If the generated story, options, image, and music in each turn, users could ask to re-generate them.
+"""
+export_phase_description = """
+In this phase, you can export the generated whole stuffs including text, image, audio, and video in the form of a static HTML page.
 """

interfaces/export_ui.py

@@ -0,0 +1,63 @@
+import gradio as gr
+from templates import parser
+from interfaces import utils
+from modules import palmchat
+
+template_file = "templates/basic.jinja"
+
+async def title_gen(cursors):
+    stories = ""
+    for cursor in cursors:
+        stories = stories + cursor["story"]
+  
+    prompt = f"""what would be the title of the story below? be specific and creative.
+
+{stories}
+
+title: """
+
+    parameters = {
+		'model': 'models/text-bison-001',
+		'candidate_count': 1,
+		'temperature': 0.7,
+		'top_k': 40,
+		'top_p': 1,
+		'max_output_tokens': 4096,
+	}    	
+    _, title = await palmchat.gen_text(prompt, mode="text", parameters=parameters)
+    return title
+
+def export(
+    title, cursors,
+	main_char_img, main_char_name, main_char_age, main_char_mbti, main_char_personality, main_char_job,
+	side_char_enable1, side_char_img1, side_char_name1, side_char_age1, side_char_mbti1, side_char_personality1, side_char_job1,
+	side_char_enable2, side_char_img2, side_char_name2, side_char_age2, side_char_mbti2, side_char_personality2, side_char_job2,
+	side_char_enable3, side_char_img3, side_char_name3, side_char_age3, side_char_mbti3, side_char_personality3, side_char_job3,    
+):
+    print(main_char_img)
+    characters = [
+        {
+            'img': main_char_img,
+            'name': main_char_name,
+        }
+    ]
+    utils.add_side_character_to_export(
+        characters, side_char_enable1, side_char_img1, side_char_name1, side_char_age1, side_char_mbti1, side_char_personality1, side_char_job1
+    )
+    utils.add_side_character_to_export(
+        characters, side_char_enable2, side_char_img2, side_char_name2, side_char_age2, side_char_mbti2, side_char_personality2, side_char_job2
+    )
+    utils.add_side_character_to_export(
+        characters, side_char_enable3, side_char_img3, side_char_name3, side_char_age3, side_char_mbti3, side_char_personality3, side_char_job3
+    )
+
+    html_as_string = parser.gen_from_file(
+        template_file,
+        kwargs={
+            "title": title,
+            "characters": characters,
+            "items": cursors
+        }
+    )
+
+    return html_as_string

interfaces/story_gen_ui.py

@@ -14,10 +14,11 @@ from pingpong import PingPong
 from pingpong.context import CtxLastWindowStrategy
 
 # TODO: Replace checkpoint filename to Huggingface URL
-img_maker = ImageMaker('landscapeAnimePro_v20Inspiration.safetensors', vae="cute20vae.safetensors")
+img_maker = ImageMaker('landscapeAnimePro_v20Inspiration.safetensors') # without VAE
+#img_maker = ImageMaker('landscapeAnimePro_v20Inspiration.safetensors', vae="stabilityai/sd-vae-ft-mse")
 #img_maker = ImageMaker('fantasyworldFp16.safetensors', vae="cute20vae.safetensors")
 #img_maker = ImageMaker('forgesagalandscapemi.safetensors', vae="anythingFp16.safetensors")
-bgm_maker = None#MusicMaker(model_size='large', output_format='mp3')
+bgm_maker = MusicMaker(model_size='medium', output_format='mp3')
 
 video_gen_client_url = "https://0447df3cf5f7c49c46.gradio.live"
 

interfaces/ui.py

@@ -7,7 +7,7 @@ import PIL
 from pathlib import Path
 
 from constants.init_values import (
-	places, moods, jobs, random_names, default_character_images
+	genres, places, moods, jobs, random_names, default_character_images
 )
 
 from modules import (
@@ -23,6 +23,9 @@ img_maker = ImageMaker('hellonijicute25d_V10b.safetensors') # without_VAE
 ############
 # for plotting
 
+def update_selected_char_image(evt: gr.EventData):
+    return evt.value
+
 def get_random_name(cur_char_name, char_name1, char_name2, char_name3):
 	tmp_random_names = copy.deepcopy(random_names)
 	tmp_random_names.remove(cur_char_name)
@@ -58,10 +61,10 @@ def gen_character_image(
 		img_filename = str(Path('.') / 'assets' / 'nsfw_warning.png')
 
 	# update gallery
-	gen_image = numpy.asarray(PIL.Image.open(img_filename))
-	gallery_images.insert(0, gen_image)
+	# gen_image = numpy.asarray(PIL.Image.open(img_filename))
+	gallery_images.insert(0, img_filename)
 
-	return gr.update(value=gallery_images), gallery_images
+	return gr.update(value=gallery_images), gallery_images, img_filename
 
 
 def update_on_age(evt: gr.SelectData): 
@@ -71,9 +74,9 @@ def update_on_age(evt: gr.SelectData):
         gr.update(value=places[evt.value][0], choices=places[evt.value]),
         gr.update(value=moods[evt.value][0], choices=moods[evt.value]),
         gr.update(value=job_list[0], choices=job_list),
-        gr.update(value=job_list[0], choices=job_list),
-        gr.update(value=job_list[0], choices=job_list),
-        gr.update(value=job_list[0], choices=job_list)
+        gr.update(value=job_list[1], choices=job_list),
+        gr.update(value=job_list[2], choices=job_list),
+        gr.update(value=job_list[3], choices=job_list)
 	)    
 
 ############
@@ -91,3 +94,67 @@ def update_on_main_tabs(chat_state, evt: gr.SelectData):
 
     ppm = chat_state[chat_mode]
     return chat_mode, ppm.build_uis()
+
+def reset():
+	from modules.palmchat import GradioPaLMChatPPManager
+    
+	return (
+		[], # cursors
+		0, # cur_cursor
+  
+		{
+				"setting_chat": GradioPaLMChatPPManager(),
+				"story_chat": GradioPaLMChatPPManager(),
+				"export_chat": GradioPaLMChatPPManager(),
+		}, # chat_state
+		"setting_chat", # chat_mode
+  
+		default_character_images, # gallery_images1
+		default_character_images, # gallery_images2
+		default_character_images, # gallery_images3
+		default_character_images, # gallery_images4
+		default_character_images[0], # selected_main_char_image1 
+		default_character_images[0], # selected_side_char_image1 
+		default_character_images[0], # selected_side_char_image2 
+		default_character_images[0], # selected_side_char_image3 
+
+		genres[0], # genre_dd
+		places[genres[0]][0], # place_dd
+		moods[genres[0]][0], # mood_dd
+
+		default_character_images, # char_gallery1
+		jobs[genres[0]][0], # job_dd1
+
+		False, # side_char_enable_ckb1
+		default_character_images, # char_gallery2
+		jobs[genres[0]][1], # job_dd2
+
+		False, # side_char_enable_ckb2
+		default_character_images, # char_gallery3
+		jobs[genres[0]][2], # job_dd3
+
+		False, # side_char_enable_ckb3
+		default_character_images, # char_gallery4
+		jobs[genres[0]][3], # job_dd4
+
+		None, # story_image
+		None, # story_audio
+		None, # story_video
+
+		'', # story_content
+		gr.Slider(
+			1, 2, 1, step=1, interactive=True, 
+			label="1/2", visible=False
+		), # story_progress
+
+		'', # custom_prompt_txt
+
+		'Action 1', # action_btn1
+		'Action 2', # action_btn2
+		'Action 3', # action_btn3
+		'', # custom_action_txt
+
+		'Your Own Story', # title_txt
+
+		"", # export_html
+	)

interfaces/utils.py

@@ -6,6 +6,20 @@ import random
 from modules import palmchat
 from pingpong.context import CtxLastWindowStrategy
 
+def add_side_character_to_export(
+	characters,	enable, img, 
+	name, age, mbti, personality, job
+):
+	if enable:
+		characters.append(
+			{
+				'img': img,
+				'name': name
+			}
+		)
+
+	return characters
+
 def add_side_character(
 	enable, prompt, cur_side_chars,
 	name, age, mbti, personality, job

modules/image_maker.py

@@ -72,8 +72,9 @@ class ImageMaker:
 
         print("Loading the Stable Diffusion model into memory...")
         self.__sd_model = StableDiffusionPipeline.from_single_file(self.model_base,
-                                                              #torch_dtype=torch.float16,
-                                                              use_safetensors=True)
+                                                              torch_dtype=torch.float16,
+                                                              use_safetensors=True,
+                                                              )
 
         # Clip Skip
         self.__sd_model.text_encoder.text_model.encoder.layers = self.__sd_model.text_encoder.text_model.encoder.layers[:12 - (self.clip_skip - 1)]
@@ -88,7 +89,7 @@ class ImageMaker:
 
         # VAE
         if self.vae:
-            vae_model = AutoencoderKL.from_single_file(self.vae)
+            vae_model = AutoencoderKL.from_pretrained(self.vae, torch_dtype=torch.float16)
             self.__sd_model.vae = vae_model
 
         if not self.safety:
@@ -243,7 +244,7 @@ class ImageMaker:
         
         try: 
             res_json = json.loads(response_txt)
-            positive = (res_json['main_sentence'] if not positive else f"{positive}, {res_json['main_sentence']}") + ", "
+            positive = (res_json['primary_sentence'] if not positive else f"{positive}, {res_json['primary_sentence']}") + ", "
             positive += ', '.join(res_json['descriptors'])
         except:
             print("=== PaLM Response ===")

modules/music_maker.py

@@ -134,7 +134,7 @@ class MusicMaker:
             print("=== PaLM Response ===")            
             raise ValueError("The response from PaLM API is not in the expected format.")
             
-        return res_json['main_sentence']
+        return res_json['primary_sentence']
 
 
     @property

pyproject.toml

@@ -23,6 +23,7 @@ bingbong = "^0.4.2"
 asyncio = "^3.4.3"
 toml = "^0.10.2"
 compel = "^2.0.2"
+jinja2 = "^3.1.2"
 
 [[tool.poetry.source]]
 name = "pytorch"

templates/basic.jinja

@@ -0,0 +1,96 @@
+<!DOCTYPE html>
+<html>
+<head>
+<style>
+
+#title {
+	color: blue;
+	text-align: center;
+	margin-bottom: 50px;
+}
+
+video, audio {
+	display: block;
+	width: 90%;
+	margin: auto !important;
+}
+
+audio {
+	padding-top: 10px !important;
+}
+
+#story-body {
+	padding-top: 20px;
+	width: 50%;
+	margin: auto;
+}
+
+#characters {
+	text-align: center;
+	display: flex;
+	width: 40%;
+	margin: auto;
+}
+
+.character {
+	float: left;
+	padding-left: 5px;
+	padding-right: 5px;
+}
+
+.character p {
+	text-align: center;
+}
+
+.paragraphs {
+	text-align: justify;
+	padding-top: 50px;
+}
+
+</style>
+</head>
+<body>
+	<h1 id="title">
+		{{ title }}
+	</h1>
+
+	<div id="characters">
+	{% for character in characters %}
+		<div class="character">
+			<img src="file/{{ character['img'] }}" />
+			<p>{{ character['name'] }}</p>
+		</div>
+	{% endfor %}
+	</div>
+
+	<div></div>
+
+	<div id="story-body">
+	{% for item in items %}
+	<div>
+	{%    if 'video' in item %}
+		<video controls>
+			<source src="file/{{ item['video'] }}" type="video/mp4">
+			Your browser does not support the video tag.
+		</video>
+	{%    else %}
+	{%        if 'img' in item %}
+		<img class="story-img" src="file/{{ item['img'] }}" />
+	{%        endif %}
+	{%        if 'audio' in item %}
+		<audio controls>
+			<source src="file/{{ item['audio'] }}" type="audio/wav">
+			Your browser does not support the audio element.
+		</audio>
+	{%        endif %}
+	{% endif %}
+
+	<p class="paragraphs">
+	{{ item['story'] }}
+	</p>
+	</div>
+	{% endfor %}
+
+	</div>
+</body>
+</html>

templates/parser.py

@@ -0,0 +1,15 @@
+import jinja2
+
+def gen_from_file(filename, kwargs):
+    # for basic template there are two keys that should be provided
+    # characters (list) 
+    # - each item has 'img' and 'name' keys
+    #
+    # items (stories)
+    # - each item has 'video', 'img', 'audio', and 'story'
+    html_template = open(filename, "r").read()
+
+    environment = jinja2.Environment()
+    template = environment.from_string(html_template)
+
+    return template.render(**kwargs)