liruiw Chen Bao commited on
Commit
114ef2f
1 Parent(s): d3f2fdb

chenbao-dev (#1)

Browse files

- fix hydra problem (fb88e588e55014ef3489d020fe4a1dea18e036ae)
- fix hydra problem (5790dee08c3b08571490e392bc12cb220a069d9b)


Co-authored-by: Chen Bao <[email protected]>

Files changed (4) hide show
  1. app.py +27 -28
  2. cliport/cfg/config.yaml +1 -1
  3. cliport/cfg/data.yaml +1 -1
  4. gensim/sim_runner.py +38 -17
app.py CHANGED
@@ -3,13 +3,13 @@ import numpy as np
3
  from tempfile import NamedTemporaryFile
4
  import copy
5
  import shapely
 
6
  from shapely.geometry import *
7
  from shapely.affinity import *
8
  from omegaconf import OmegaConf
9
  from moviepy.editor import ImageSequenceClip
10
  import gradio as gr
11
 
12
-
13
  from consts import ALL_BLOCKS, ALL_BOWLS
14
  from md_logger import MarkdownLogger
15
 
@@ -37,12 +37,17 @@ from gensim.sim_runner import SimulationRunner
37
  from gensim.memory import Memory
38
  from gensim.utils import set_gpt_model, clear_messages
39
 
 
40
  class DemoRunner:
41
-
42
  def __init__(self):
43
  self._env = None
 
 
 
44
 
45
  def setup(self, api_key):
 
46
  openai.api_key = api_key
47
  cfg['model_output_dir'] = 'temp'
48
  cfg['prompt_folder'] = 'topdown_task_generation_prompt_simple_singleprompt'
@@ -56,48 +61,43 @@ class DemoRunner:
56
  self.simulation_runner = SimulationRunner(cfg, agent, critic, memory)
57
 
58
  info = '### Build'
59
- img = np.zeros((720, 640, 0))
60
 
61
  return info, img
62
 
63
  def run(self, instruction):
 
64
  cfg['target_task_name'] = instruction
65
 
66
- self._env.cache_video = []
67
- self._md_logger.clear()
68
-
69
- try:
70
- self.simulation_runner.task_creation()
71
- self.simulation_runner.simulate_task()
72
- except Exception as e:
73
- return f'Error: {e}', None, None
74
-
75
- video_file_name = None
76
- if self._env.cache_video:
77
- rendered_clip = ImageSequenceClip(self._env.cache_video, fps=25)
78
- video_file_name = NamedTemporaryFile(suffix='.mp4').name
79
- rendered_clip.write_videofile(video_file_name, fps=25)
80
 
81
- return self.simulation_runner.chat_log, self.simulation_runner.env.curr_video, video_file_name
 
 
 
82
 
83
 
84
  def setup(api_key):
85
  if not api_key:
86
  return 'Please enter your OpenAI API key!', None, None
87
-
88
  demo_runner = DemoRunner()
89
-
90
  info, img = demo_runner.setup(api_key)
91
  return info, img, demo_runner
92
 
93
 
94
  def run(instruction, demo_runner):
95
  if demo_runner is None:
96
- return 'Please run setup first!', None, None
 
 
97
  return demo_runner.run(instruction)
98
 
99
 
100
  if __name__ == '__main__':
 
101
  with open('README.md', 'r') as f:
102
  for _ in range(12):
103
  next(f)
@@ -120,22 +120,21 @@ if __name__ == '__main__':
120
 
121
  with gr.Row():
122
  with gr.Column():
123
-
124
  inp_instruction = gr.Textbox(label='Task Name', lines=1)
125
  btn_run = gr.Button("Run (this may take 30+ seconds)")
126
  info_run = gr.Markdown(label='Generated Code')
127
  with gr.Column():
128
  video_run = gr.Video(label='Video of Last Instruction')
129
-
130
  btn_setup.click(
131
- setup,
132
  inputs=[inp_api_key],
133
  outputs=[info_setup, img_setup, state]
134
  )
135
  btn_run.click(
136
- run,
137
- inputs=[inp_instruction, state],
138
- outputs=[info_run, img_setup, video_run]
139
  )
140
-
141
  demo.queue().launch(show_error=True)
 
3
  from tempfile import NamedTemporaryFile
4
  import copy
5
  import shapely
6
+ from hydra.core.global_hydra import GlobalHydra
7
  from shapely.geometry import *
8
  from shapely.affinity import *
9
  from omegaconf import OmegaConf
10
  from moviepy.editor import ImageSequenceClip
11
  import gradio as gr
12
 
 
13
  from consts import ALL_BLOCKS, ALL_BOWLS
14
  from md_logger import MarkdownLogger
15
 
 
37
  from gensim.memory import Memory
38
  from gensim.utils import set_gpt_model, clear_messages
39
 
40
+
41
  class DemoRunner:
42
+
43
  def __init__(self):
44
  self._env = None
45
+ GlobalHydra.instance().clear()
46
+ hydra.initialize(version_base="1.2", config_path='cliport/cfg')
47
+ self._cfg = hydra.compose(config_name="data")
48
 
49
  def setup(self, api_key):
50
+ cfg = self._cfg
51
  openai.api_key = api_key
52
  cfg['model_output_dir'] = 'temp'
53
  cfg['prompt_folder'] = 'topdown_task_generation_prompt_simple_singleprompt'
 
61
  self.simulation_runner = SimulationRunner(cfg, agent, critic, memory)
62
 
63
  info = '### Build'
64
+ img = np.zeros((720, 640, 3))
65
 
66
  return info, img
67
 
68
  def run(self, instruction):
69
+ cfg = self._cfg
70
  cfg['target_task_name'] = instruction
71
 
72
+ # self._env.cache_video = []
73
+ self.simulation_runner._md_logger = ''
 
 
 
 
 
 
 
 
 
 
 
 
74
 
75
+ self.simulation_runner.task_creation()
76
+ self.simulation_runner.simulate_task()
77
+ print("self.video_path = ", self.simulation_runner.video_path)
78
+ return self.simulation_runner._md_logger, self.simulation_runner.video_path
79
 
80
 
81
  def setup(api_key):
82
  if not api_key:
83
  return 'Please enter your OpenAI API key!', None, None
84
+
85
  demo_runner = DemoRunner()
86
+
87
  info, img = demo_runner.setup(api_key)
88
  return info, img, demo_runner
89
 
90
 
91
  def run(instruction, demo_runner):
92
  if demo_runner is None:
93
+ return 'Please run setup first!', None
94
+
95
+ # return None, "/home/baochen/Desktop/projects/GenSim2/data/assemble-pallet-ball-train/videos/000001.mp4"
96
  return demo_runner.run(instruction)
97
 
98
 
99
  if __name__ == '__main__':
100
+ os.environ['GENSIM_ROOT'] = os.getcwd()
101
  with open('README.md', 'r') as f:
102
  for _ in range(12):
103
  next(f)
 
120
 
121
  with gr.Row():
122
  with gr.Column():
 
123
  inp_instruction = gr.Textbox(label='Task Name', lines=1)
124
  btn_run = gr.Button("Run (this may take 30+ seconds)")
125
  info_run = gr.Markdown(label='Generated Code')
126
  with gr.Column():
127
  video_run = gr.Video(label='Video of Last Instruction')
128
+
129
  btn_setup.click(
130
+ setup,
131
  inputs=[inp_api_key],
132
  outputs=[info_setup, img_setup, state]
133
  )
134
  btn_run.click(
135
+ run,
136
+ inputs=[inp_instruction, state],
137
+ outputs=[info_run, video_run]
138
  )
139
+
140
  demo.queue().launch(show_error=True)
cliport/cfg/config.yaml CHANGED
@@ -5,7 +5,7 @@ tag: default
5
  debug: False
6
  gpt_temperature: 0.8 # GPT-4 response temperature. higher means more diversity
7
  prompt_folder: vanilla_task_generation_prompt # the prompt folder that stores the prompt chain
8
- max_env_run_cnt: 3 # maximum number of runs for each environment
9
  trials: 10 # how many times of spawning each environment generated
10
  output_folder: 'output/output_stats'
11
  model_output_dir: '' # to be filled in with date
 
5
  debug: False
6
  gpt_temperature: 0.8 # GPT-4 response temperature. higher means more diversity
7
  prompt_folder: vanilla_task_generation_prompt # the prompt folder that stores the prompt chain
8
+ max_env_run_cnt: 1 # maximum number of runs for each environment
9
  trials: 10 # how many times of spawning each environment generated
10
  output_folder: 'output/output_stats'
11
  model_output_dir: '' # to be filled in with date
cliport/cfg/data.yaml CHANGED
@@ -25,7 +25,7 @@ dataset:
25
 
26
  # record videos (super slow)
27
  record:
28
- save_video: False
29
  save_video_path: ${data_dir}/${task}-${mode}/videos/
30
  add_text: False
31
  add_task_text: True
 
25
 
26
  # record videos (super slow)
27
  record:
28
+ save_video: True
29
  save_video_path: ${data_dir}/${task}-${mode}/videos/
30
  add_text: False
31
  add_task_text: True
gensim/sim_runner.py CHANGED
@@ -22,8 +22,10 @@ from gensim.utils import (
22
  )
23
  import pybullet as p
24
 
 
25
  class SimulationRunner:
26
  """ the main class that runs simulation loop """
 
27
  def __init__(self, cfg, agent, critic, memory):
28
  self.cfg = cfg
29
  self.agent = agent
@@ -46,19 +48,25 @@ class SimulationRunner:
46
  self.generated_task_programs = []
47
  self.generated_task_names = []
48
  self.generated_tasks = []
49
- self.passed_tasks = [] # accepted ones
 
 
 
50
 
51
  def print_current_stats(self):
52
  """ print the current statistics of the simulation design """
53
  print("=========================================================")
54
- print(f"{self.cfg['prompt_folder']} Trial {self.curr_trials} SYNTAX_PASS_RATE: {(self.syntax_pass_rate / (self.curr_trials)) * 100:.1f}% RUNTIME_PASS_RATE: {(self.runtime_pass_rate / (self.curr_trials)) * 100:.1f}% ENV_PASS_RATE: {(self.env_pass_rate / (self.curr_trials)) * 100:.1f}%")
 
55
  print("=========================================================")
56
 
57
  def save_stats(self):
58
  """ save the final simulation statistics """
59
  self.diversity_score = compute_diversity_score_from_assets(self.task_asset_logs, self.curr_trials)
60
- save_stat(self.cfg, self.cfg['model_output_dir'], self.generated_tasks, self.syntax_pass_rate / (self.curr_trials),
61
- self.runtime_pass_rate / (self.curr_trials), self.env_pass_rate / (self.curr_trials), self.diversity_score)
 
 
62
  print("Model Folder: ", self.cfg['model_output_dir'])
63
  print(f"Total {len(self.generated_tasks)} New Tasks:", [task['task-name'] for task in self.generated_tasks])
64
  try:
@@ -66,7 +74,6 @@ class SimulationRunner:
66
  except:
67
  pass
68
 
69
-
70
  def task_creation(self):
71
  """ create the task through interactions of agent and critic """
72
  self.task_creation_pass = True
@@ -93,16 +100,15 @@ class SimulationRunner:
93
  # self.curr_task_name = self.generated_task['task-name']
94
  print("task creation time {:.3f}".format(time.time() - start_time))
95
 
96
-
97
  def setup_env(self):
98
  """ build the new task"""
99
  env = Environment(
100
- self.cfg['assets_root'],
101
- disp=self.cfg['disp'],
102
- shared_memory=self.cfg['shared_memory'],
103
- hz=480,
104
- record_cfg=self.cfg['record']
105
- )
106
 
107
  task = eval(self.curr_task_name)()
108
  task.mode = self.cfg['mode']
@@ -118,15 +124,15 @@ class SimulationRunner:
118
  print(f"Mode: {task.mode}")
119
 
120
  # Start video recording
121
- if record:
122
- env.start_rec(f'{dataset.n_episodes+1:06d}')
123
 
124
  return task, dataset, env, expert
125
 
126
  def run_one_episode(self, dataset, expert, env, task, episode, seed):
127
  """ run the new task for one episode """
128
  add_to_txt(
129
- self.chat_log, f"================= TRIAL: {self.curr_trials}", with_print=True)
130
  record = self.cfg['record']['save_video']
131
  np.random.seed(seed)
132
  random.seed(seed)
@@ -138,6 +144,12 @@ class SimulationRunner:
138
  reward = 0
139
  total_reward = 0
140
 
 
 
 
 
 
 
141
  # Rollout expert policy
142
  for _ in range(task.max_steps):
143
  act = expert.act(obs, info)
@@ -149,6 +161,12 @@ class SimulationRunner:
149
  if done:
150
  break
151
 
 
 
 
 
 
 
152
  episode.append((obs, None, reward, info))
153
  return total_reward
154
 
@@ -159,10 +177,10 @@ class SimulationRunner:
159
  env_success_cnt = 0.
160
  seed = 123
161
  self.curr_trials += 1
162
-
163
  if p.isConnected():
164
  p.disconnect()
165
-
166
  if not self.task_creation_pass:
167
  print("task creation failure => count as syntax exceptions.")
168
  return
@@ -179,6 +197,7 @@ class SimulationRunner:
179
  save_text(self.cfg['model_output_dir'], self.generated_task_name + '_error', str(traceback.format_exc()))
180
  print("========================================================")
181
  print("Syntax Exception:", to_print)
 
182
  return
183
 
184
  try:
@@ -201,4 +220,6 @@ class SimulationRunner:
201
  save_text(self.cfg['model_output_dir'], self.generated_task_name + '_error', str(traceback.format_exc()))
202
  print("========================================================")
203
  print("Runtime Exception:", to_print)
 
 
204
  self.memory.save_run(self.generated_task)
 
22
  )
23
  import pybullet as p
24
 
25
+
26
  class SimulationRunner:
27
  """ the main class that runs simulation loop """
28
+
29
  def __init__(self, cfg, agent, critic, memory):
30
  self.cfg = cfg
31
  self.agent = agent
 
48
  self.generated_task_programs = []
49
  self.generated_task_names = []
50
  self.generated_tasks = []
51
+ self.passed_tasks = [] # accepted ones
52
+
53
+ self.video_path = None
54
+ self._md_logger = ''
55
 
56
  def print_current_stats(self):
57
  """ print the current statistics of the simulation design """
58
  print("=========================================================")
59
+ print(
60
+ f"{self.cfg['prompt_folder']} Trial {self.curr_trials} SYNTAX_PASS_RATE: {(self.syntax_pass_rate / (self.curr_trials)) * 100:.1f}% RUNTIME_PASS_RATE: {(self.runtime_pass_rate / (self.curr_trials)) * 100:.1f}% ENV_PASS_RATE: {(self.env_pass_rate / (self.curr_trials)) * 100:.1f}%")
61
  print("=========================================================")
62
 
63
  def save_stats(self):
64
  """ save the final simulation statistics """
65
  self.diversity_score = compute_diversity_score_from_assets(self.task_asset_logs, self.curr_trials)
66
+ save_stat(self.cfg, self.cfg['model_output_dir'], self.generated_tasks,
67
+ self.syntax_pass_rate / (self.curr_trials),
68
+ self.runtime_pass_rate / (self.curr_trials), self.env_pass_rate / (self.curr_trials),
69
+ self.diversity_score)
70
  print("Model Folder: ", self.cfg['model_output_dir'])
71
  print(f"Total {len(self.generated_tasks)} New Tasks:", [task['task-name'] for task in self.generated_tasks])
72
  try:
 
74
  except:
75
  pass
76
 
 
77
  def task_creation(self):
78
  """ create the task through interactions of agent and critic """
79
  self.task_creation_pass = True
 
100
  # self.curr_task_name = self.generated_task['task-name']
101
  print("task creation time {:.3f}".format(time.time() - start_time))
102
 
 
103
  def setup_env(self):
104
  """ build the new task"""
105
  env = Environment(
106
+ self.cfg['assets_root'],
107
+ disp=self.cfg['disp'],
108
+ shared_memory=self.cfg['shared_memory'],
109
+ hz=480,
110
+ record_cfg=self.cfg['record']
111
+ )
112
 
113
  task = eval(self.curr_task_name)()
114
  task.mode = self.cfg['mode']
 
124
  print(f"Mode: {task.mode}")
125
 
126
  # Start video recording
127
+ # if record:
128
+ # env.start_rec(f'{dataset.n_episodes+1:06d}')
129
 
130
  return task, dataset, env, expert
131
 
132
  def run_one_episode(self, dataset, expert, env, task, episode, seed):
133
  """ run the new task for one episode """
134
  add_to_txt(
135
+ self.chat_log, f"================= TRIAL: {self.curr_trials}", with_print=True)
136
  record = self.cfg['record']['save_video']
137
  np.random.seed(seed)
138
  random.seed(seed)
 
144
  reward = 0
145
  total_reward = 0
146
 
147
+ save_data = self.cfg['save_data']
148
+ # Start recording video (NOTE: super slow)
149
+ if record:
150
+ video_name = f'{dataset.n_episodes + 1:06d}'
151
+ env.start_rec(video_name)
152
+
153
  # Rollout expert policy
154
  for _ in range(task.max_steps):
155
  act = expert.act(obs, info)
 
161
  if done:
162
  break
163
 
164
+ # End recording video
165
+ if record:
166
+ env.end_rec()
167
+ self.video_path = os.path.join(self.cfg['record']['save_video_path'],
168
+ f"{video_name}.mp4")
169
+
170
  episode.append((obs, None, reward, info))
171
  return total_reward
172
 
 
177
  env_success_cnt = 0.
178
  seed = 123
179
  self.curr_trials += 1
180
+
181
  if p.isConnected():
182
  p.disconnect()
183
+
184
  if not self.task_creation_pass:
185
  print("task creation failure => count as syntax exceptions.")
186
  return
 
197
  save_text(self.cfg['model_output_dir'], self.generated_task_name + '_error', str(traceback.format_exc()))
198
  print("========================================================")
199
  print("Syntax Exception:", to_print)
200
+ self._md_logger = str(traceback.format_exc())
201
  return
202
 
203
  try:
 
220
  save_text(self.cfg['model_output_dir'], self.generated_task_name + '_error', str(traceback.format_exc()))
221
  print("========================================================")
222
  print("Runtime Exception:", to_print)
223
+ self._md_logger = str(traceback.format_exc())
224
+
225
  self.memory.save_run(self.generated_task)