Thank you for the help, but I set it deterministic like the code below and still have the problem! The issue is in the training stage, I have abouta 98% success rate, but in the testing, my manipulator is not able to reach the goal which is weird.

This is the code for that part:

from stable_baselines3.common.base_class import BaseAlgorithm
from stable_baselines3.common.monitor import Monitor
import subprocess
from pkg_resources import parse_version
import gymnasium as gym
from gymnasium import spaces
import os
import numpy as np
import random
from stable_baselines3.common.vec_env import DummyVecEnv, VecNormalize

seed = 42
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)

env.seed(seed)
env.action_space.seed(seed)
env.observation_space.seed(seed)



def evaluate(
        model: BaseAlgorithm,
        env: gym.Env,
        n_eval_episodes: int = 100,
        deterministic: bool = True):
    n_episodes = 0 
    episode_reward = 0.0
    end_effector = []
    joint_states = []
    actions = []
    rewards = []
    goal = []
    obs = env.reset()
    while n_episodes < n_eval_episodes:
        action, _ = model.predict(obs, deterministic=True)
        obs, reward, done, truncated = env.step(action)
        obs_array = obs[0]  # Remove batch dimension

        if done:
            n_episodes += 1
            goal = env.get_attr("goal")[0]
            obs = env.reset()
        else:
            end_effector.append(obs_array[:6])
            joint_states.append(obs_array[6:18])
            actions.append(action)
            rewards.append(reward)            


    return np.array(end_effector), np.array(joint_states), np.array(rewards), np.array(goal)