update

Model-free Control/.idea/.gitignore

@@ -0,0 +1,3 @@
+
+# Default ignored files
+/workspace.xml

Model-free Control/.idea/Model-free Control.iml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.7" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="Unittests" />
+  </component>
+</module>

Model-free Control/.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

Model-free Control/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7" project-jdk-type="Python SDK" />
+</project>

Model-free Control/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/Model-free Control.iml" filepath="$PROJECT_DIR$/.idea/Model-free Control.iml" />
+    </modules>
+  </component>
+</project>

Model-free Control/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
+  </component>
+</project>

Model-free Control/env.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+@author: huiming zhou
+"""
+
+x_range, y_range = 51, 31     # size of background
+
+
+def obs_map():
+    """
+    Initialize obstacles' positions
+
+    :return: map of obstacles
+    """
+
+    obs = []
+    for i in range(x_range):
+        obs.append((i, 0))
+    for i in range(x_range):
+        obs.append((i, y_range - 1))
+
+    for i in range(y_range):
+        obs.append((0, i))
+    for i in range(y_range):
+        obs.append((x_range - 1, i))
+
+    for i in range(10, 21):
+        obs.append((i, 15))
+    for i in range(15):
+        obs.append((20, i))
+
+    for i in range(15, 30):
+        obs.append((30, i))
+    for i in range(16):
+        obs.append((40, i))
+
+    return obs
+
+
+def lose_map():
+    """
+    Initialize losing states' positions
+    :return: losing states
+    """
+
+    lose = []
+    for i in range(25, 36):
+        lose.append((i, 13))
+
+    return lose
+
+
+def get_reward(x_next, xG, lose):
+    """
+    calculate reward of next state
+
+    :param x_next: next state
+    :return: reward
+    """
+
+    reward = []
+    for x in x_next:
+        if x in xG:
+            reward.append(10)           # reward : 10, for goal states
+        elif x in lose:
+            reward.append(-10)          # reward : -10, for lose states
+        else:
+            reward.append(0)            # reward : 0, for other states
+
+    return reward

Model-free Control/motion_model.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+@author: huiming zhou
+"""
+import numpy as np
+
+motions = [(1, 0), (-1, 0), (0, 1), (0, -1)]                # feasible motion sets
+
+def move_prob(x, u, obs, eta = 0.2):
+    """
+    Motion model of robots,
+
+    :param x: current state (node)
+    :param u: input
+    :param obs: obstacle map
+    :param eta: noise in motion model
+    :return: next states and corresponding probability
+    """
+
+    p_next = [1 - eta, eta / 2, eta / 2]
+    x_next = []
+    if u == (0, 1):
+        u_real = [(0, 1), (-1, 0), (1, 0)]
+    elif u == (0, -1):
+        u_real = [(0, -1), (-1, 0), (1, 0)]
+    elif u == (-1, 0):
+        u_real = [(-1, 0), (0, 1), (0, -1)]
+    else:
+        u_real = [(1, 0), (0, 1), (0, -1)]
+
+    for act in u_real:
+        if (x[0] + act[0], x[1] + act[1]) in obs:
+            x_next.append(x)
+        else:
+            x_next.append((x[0] + act[0], x[1] + act[1]))
+
+    return x_next, p_next

Model-free Control/tools.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+@author: huiming zhou
+"""
+
+import matplotlib.pyplot as plt
+
+
+def extract_path(xI, xG, parent, actions):
+    """
+    Extract the path based on the relationship of nodes.
+
+    :param xI: Starting node
+    :param xG: Goal node
+    :param parent: Relationship between nodes
+    :param actions: Action needed for transfer between two nodes
+    :return: The planning path
+    """
+
+    path_back = [xG]
+    acts_back = [actions[xG]]
+    x_current = xG
+    while True:
+        x_current = parent[x_current]
+        path_back.append(x_current)
+        acts_back.append(actions[x_current])
+        if x_current == xI: break
+
+    return list(reversed(path_back)), list(reversed(acts_back))
+
+
+def showPath(xI, xG, path):
+    """
+    Plot the path.
+
+    :param xI: Starting node
+    :param xG: Goal node
+    :param path: Planning path
+    :return: A plot
+    """
+
+    path.remove(xI)
+    path.remove(xG)
+    path_x = [path[i][0] for i in range(len(path))]
+    path_y = [path[i][1] for i in range(len(path))]
+    plt.plot(path_x, path_y, linewidth='5', color='r', linestyle='-')
+    plt.pause(0.001)
+    plt.show()
+
+
+def show_map(xI, xG, obs_map, lose_map, name):
+    """
+    Plot the background you designed.
+
+    :param xI: starting state
+    :param xG: goal states
+    :param obs_map: positions of obstacles
+    :param lose_map: positions of losing state
+    :param name: name of this figure
+    :return: a figure
+    """
+
+    obs_x = [obs_map[i][0] for i in range(len(obs_map))]
+    obs_y = [obs_map[i][1] for i in range(len(obs_map))]
+
+    lose_x = [lose_map[i][0] for i in range(len(lose_map))]
+    lose_y = [lose_map[i][1] for i in range(len(lose_map))]
+
+    plt.plot(xI[0], xI[1], "bs")                                    # plot starting state (blue)
+
+    for x in xG:
+        plt.plot(x[0], x[1], "gs")                                  # plot goal states (green)
+
+    plt.plot(obs_x, obs_y, "sk")                                    # plot obstacles (black)
+    plt.plot(lose_x, lose_y, marker = 's', color = '#A52A2A')       # plot losing states (grown)
+    plt.title(name, fontdict=None)
+    plt.axis("equal")
+
+
+def plot_dots(x):
+    """
+    Plot state x for animation
+
+    :param x: current node
+    :return: a plot
+    """
+
+    plt.plot(x[0], x[1], linewidth='3', color='#808080', marker='o')    # plot dots for animation
+    plt.gcf().canvas.mpl_connect('key_release_event',
+                                 lambda event: [exit(0) if event.key == 'escape' else None])
+    plt.pause(0.001)
+
+

Search-based Planning/.idea/workspace.xml

@@ -2,13 +2,7 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="025aff36-a6aa-4945-ab7e-b2c625055f47" name="Default Changelist" comment="">
-      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../Stochastic Shortest Path/Q-policy_iteration.py" beforeDir="false" afterPath="$PROJECT_DIR$/../Stochastic Shortest Path/Q-policy_iteration.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../Stochastic Shortest Path/Q-value_iteration.py" beforeDir="false" afterPath="$PROJECT_DIR$/../Stochastic Shortest Path/Q-value_iteration.py" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/../Stochastic Shortest Path/env.py" beforeDir="false" afterPath="$PROJECT_DIR$/../Stochastic Shortest Path/env.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../Stochastic Shortest Path/policy_iteration.py" beforeDir="false" afterPath="$PROJECT_DIR$/../Stochastic Shortest Path/policy_iteration.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../Stochastic Shortest Path/tools.py" beforeDir="false" afterPath="$PROJECT_DIR$/../Stochastic Shortest Path/tools.py" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/../Stochastic Shortest Path/value_iteration.py" beforeDir="false" afterPath="$PROJECT_DIR$/../Stochastic Shortest Path/value_iteration.py" afterDir="false" />
     </list>
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />

Stochastic Shortest Path/env.py

@@ -47,7 +47,7 @@ def lose_map():
     lose = []
     for i in range(25, 36):
         lose.append((i, 13))
-        
+
     return lose