lightrft.trainer

• lightrft.trainer.experience_maker
  • Experience
    • Experience.action_entropy
    • Experience.action_log_probs
    • Experience.action_mask
    • Experience.advantages
    • Experience.attention_mask
    • Experience.base_action_log_probs
    • Experience.info
    • Experience.kl
    • Experience.labels
    • Experience.pin_memory()
    • Experience.references
    • Experience.returns
    • Experience.sequences
    • Experience.to_device()
    • Experience.values
  • NaiveExperienceMaker
    • NaiveExperienceMaker.generate_samples()
    • NaiveExperienceMaker.get_advantages_and_returns()
    • NaiveExperienceMaker.get_cumulative_returns()
    • NaiveExperienceMaker.make_experience()
    • NaiveExperienceMaker.make_experience_list()
    • NaiveExperienceMaker.process_experiences()
    • NaiveExperienceMaker.tokenize_fn()
  • Samples
    • Samples.action_mask
    • Samples.attention_mask
    • Samples.labels
    • Samples.num_actions
    • Samples.packed_seq_lens
    • Samples.pad_len
    • Samples.prompts
    • Samples.references
    • Samples.response_length
    • Samples.sequences
    • Samples.total_length
  • clip_filter_like_weight_func()
  • pin_memory()
  • to()
• lightrft.trainer.experience_maker_vl
  • ExperienceVL
    • ExperienceVL.action_entropy
    • ExperienceVL.action_log_probs
    • ExperienceVL.action_mask
    • ExperienceVL.advantages
    • ExperienceVL.attention_mask
    • ExperienceVL.base_action_log_probs
    • ExperienceVL.image_grid_thws
    • ExperienceVL.info
    • ExperienceVL.kl
    • ExperienceVL.labels
    • ExperienceVL.pin_memory()
    • ExperienceVL.pixel_values
    • ExperienceVL.pixel_values_videos
    • ExperienceVL.raw_images
    • ExperienceVL.references
    • ExperienceVL.returns
    • ExperienceVL.sequences
    • ExperienceVL.to_device()
    • ExperienceVL.values
    • ExperienceVL.video_grid_thws
  • NaiveExperienceMakerVL
    • NaiveExperienceMakerVL.generate_samples()
    • NaiveExperienceMakerVL.get_advantages_and_returns()
    • NaiveExperienceMakerVL.get_cumulative_returns()
    • NaiveExperienceMakerVL.make_experience()
    • NaiveExperienceMakerVL.make_experience_list()
    • NaiveExperienceMakerVL.process_experiences()
    • NaiveExperienceMakerVL.processor_fn()
    • NaiveExperienceMakerVL.tokenize_fn()
  • SamplesVL
    • SamplesVL.action_mask
    • SamplesVL.attention_mask
    • SamplesVL.image_grid_thws
    • SamplesVL.image_num
    • SamplesVL.labels
    • SamplesVL.num_actions
    • SamplesVL.output_texts
    • SamplesVL.packed_seq_lens
    • SamplesVL.pixel_values
    • SamplesVL.pixel_values_videos
    • SamplesVL.prompts
    • SamplesVL.raw_images
    • SamplesVL.references
    • SamplesVL.response_length
    • SamplesVL.sequences
    • SamplesVL.total_length
    • SamplesVL.video_grid_thws
    • SamplesVL.video_num
  • cumulative_product()
  • pin_memory()
  • to()
• lightrft.trainer.fast_exp_maker
  • FastExperienceMaker
    • FastExperienceMaker
• lightrft.trainer.grm_trainer_vl
  • GRMTrainerVL
    • GRMTrainerVL.evaluate()
    • GRMTrainerVL.fit()
    • GRMTrainerVL.save_logs_and_checkpoints()
• lightrft.trainer.kl_controller
  • AdaptiveKLController
    • AdaptiveKLController.update()
  • FixedKLController
    • FixedKLController.update()
• lightrft.trainer.ppo_trainer
  • PPOTrainer
    • PPOTrainer.fit()
    • PPOTrainer.ppo_train()
    • PPOTrainer.save_logs_and_checkpoints()
    • PPOTrainer.training_step()
    • PPOTrainer.training_step_actor()
    • PPOTrainer.training_step_critic()
• lightrft.trainer.ppo_trainer_vl
  • PPOTrainerVL
    • PPOTrainerVL.evaluate()
    • PPOTrainerVL.fit()
    • PPOTrainerVL.ppo_train()
    • PPOTrainerVL.save_logs_and_checkpoints()
    • PPOTrainerVL.training_step()
    • PPOTrainerVL.training_step_actor()
    • PPOTrainerVL.training_step_critic()
• lightrft.trainer.replay_buffer
  • NaiveReplayBuffer
    • NaiveReplayBuffer.append()
    • NaiveReplayBuffer.clear()
    • NaiveReplayBuffer.collate_fn()
    • NaiveReplayBuffer.normalize()
    • NaiveReplayBuffer.sample()
• lightrft.trainer.replay_buffer_utils
  • BufferItem
    • BufferItem.action_entropy
    • BufferItem.action_log_probs
    • BufferItem.action_mask
    • BufferItem.advantages
    • BufferItem.attention_mask
    • BufferItem.base_action_log_probs
    • BufferItem.info
    • BufferItem.returns
    • BufferItem.sequences
    • BufferItem.values
  • BufferItemVL
    • BufferItemVL.action_entropy
    • BufferItemVL.action_log_probs
    • BufferItemVL.action_mask
    • BufferItemVL.advantages
    • BufferItemVL.attention_mask
    • BufferItemVL.base_action_log_probs
    • BufferItemVL.image_grid_thws
    • BufferItemVL.info
    • BufferItemVL.pixel_values
    • BufferItemVL.pixel_values_videos
    • BufferItemVL.raw_images
    • BufferItemVL.returns
    • BufferItemVL.sequences
    • BufferItemVL.values
    • BufferItemVL.video_grid_thws
  • is_vl_experience()
  • make_experience_batch()
  • remove_padding_in_sequences()
  • split_experience_batch()
  • zero_pad_sequences()
• lightrft.trainer.replay_buffer_vl
  • NaiveReplayBufferVL
    • NaiveReplayBufferVL.append()
    • NaiveReplayBufferVL.clear()
    • NaiveReplayBufferVL.collate_fn()
    • NaiveReplayBufferVL.normalize()
    • NaiveReplayBufferVL.sample()
• lightrft.trainer.spmd_ppo_trainer
  • SPMDPPOTrainerBase
    • SPMDPPOTrainerBase
  • SPMDPPOTrainer
    • SPMDPPOTrainer
  • SPMDPPOTrainerVL
    • SPMDPPOTrainerVL
• lightrft.trainer.srm_trainer_al
  • SRMTrainerAL
    • SRMTrainerAL.concatenated_forward()
    • SRMTrainerAL.concatenated_inputs()
    • SRMTrainerAL.evaluate()
    • SRMTrainerAL.fit()
    • SRMTrainerAL.save_logs_and_checkpoints()
• lightrft.trainer.srm_trainer_vl
  • SRMTrainerVL
    • SRMTrainerVL.concatenated_forward()
    • SRMTrainerVL.concatenated_inputs()
    • SRMTrainerVL.evaluate()
    • SRMTrainerVL.fit()
    • SRMTrainerVL.save_logs_and_checkpoints()
• lightrft.trainer.utils
  • RunningMoments
    • RunningMoments.update()
  • compute_clip_fraction()
  • fire_sampling()
  • get_cpgd_advantages_returns()
  • vllm_ge_0130()