An implementation of CC-POMCP

.gitignore

@@ -136,3 +136,8 @@ Thumbs.db
 *.pg
 
 .DS_Store
+
+# Cython debugging files
+########################
+*.c
+*.html

pomdp_py/__init__.py

@@ -2,6 +2,7 @@
 
 # Framework
 from pomdp_py.framework.basics import *
+from pomdp_py.framework.generalization import *
 from pomdp_py.framework.oopomdp import *
 from pomdp_py.framework.planner import *
 
@@ -22,6 +23,10 @@
 # Algorithms
 from pomdp_py.algorithms.value_iteration import ValueIteration  # Cython compiled
 from pomdp_py.algorithms.value_function import value, qvalue, belief_update
+from pomdp_py.algorithms.ccpomcp import (
+    CostModel,
+    CCPOMCP,
+)
 from pomdp_py.algorithms.pomcp import POMCP
 from pomdp_py.algorithms.po_rollout import PORollout
 from pomdp_py.algorithms.po_uct import (
@@ -38,3 +43,4 @@
 # Templates & Utilities
 from pomdp_py.utils.templates import *
 from pomdp_py.utils.debugging import TreeDebugger
+from pomdp_py.utils.cvec import Vector

pomdp_py/__main__.py

@@ -1,6 +1,6 @@
 import argparse
 
-available_problems = ["tiger", "rocksample", "mos", "tag", "load_unload"]
+available_problems = ["tiger", "rocksample", "mos", "tag", "load_unload", "ccrocksample"]
 
 
 def parse_args():
@@ -45,6 +45,11 @@ def parse_args():
 
             main()
 
+        elif args.run.lower() == "ccrocksample":
+            from pomdp_py.problems.cc_rocksample.cc_rocksample_problem import main
+
+            main()
+
         else:
             print("Unrecognized pomdp: {}".format(args.run))
 

pomdp_py/algorithms/ccpomcp.pxd

@@ -0,0 +1,66 @@
+# cython: language_level=3
+
+from pomdp_py.algorithms.po_uct cimport QNode
+from pomdp_py.algorithms.pomcp cimport POMCP, VNode
+from pomdp_py.framework.basics cimport PolicyModel, Action, Agent, State, Observation
+from pomdp_py.framework.generalization cimport Response
+from pomdp_py.utils.cvec cimport Vector
+
+
+cdef class CostModel:
+    pass
+
+
+cdef class CCQNode(QNode):
+    cdef Vector _cost_value
+    cdef Vector _avg_cost_value
+
+
+cdef class _CCPolicyActionData:
+    cdef double _prob
+    cdef Vector _cost_value
+    cdef Vector _avg_cost_value
+
+
+cdef class _CCPolicyModel(PolicyModel):
+    cdef dict[Action, _CCPolicyActionData] _data
+    cdef double _prob_sum
+
+    cdef bint _total_prob_is_not_one(_CCPolicyModel self)
+    cpdef void add(_CCPolicyModel self, Action action, double prob, CCQNode node)
+    cpdef void clear(_CCPolicyModel self)
+    cpdef Vector action_avg_cost(_CCPolicyModel self, Action action)
+    cpdef Vector action_cost_value(_CCPolicyModel self, Action action)
+    cdef public float probability(_CCPolicyModel self, Action action, State state)
+    cdef public Action sample(_CCPolicyModel self, State state)
+
+
+cdef class CCPOMCP(POMCP):
+    cdef double _r_diff
+    cdef double _tau
+    cdef double _alpha_n
+    cdef Vector _lambda
+    cdef Vector _cost_constraint
+    cdef Response _null_response
+    cdef bint _use_random_lambda
+    cdef bint _clip_lambda
+    cdef double _nu
+    cdef list[float] _cost_value_init
+    cdef unsigned int _n_constraints
+    # Buffers
+    cdef Vector _Q_lambda, _Action_UCB
+    cdef _CCPolicyModel _greedy_policy_model
+
+    cpdef public Action plan(CCPOMCP self, Agent agent)
+    cpdef QNode _create_qnode(self, tuple qnode_params = *)
+    cpdef void _greedy_policy(CCPOMCP self, VNode vnode, double explore_const, double nu)
+    cdef void _init_lambda_fn(CCPOMCP self)
+    cpdef tuple[State, Observation, Response] _sample_generative_model(CCPOMCP self, State state, Action action)
+    cpdef _search(CCPOMCP self)
+    cpdef Response _simulate(CCPOMCP self, State state, tuple history, VNode root, QNode parent,
+                             Observation observation, int depth)
+    cdef void _update_cost_constraint(CCPOMCP self, Action sampled_action)
+
+
+cdef double _compute_visits_ratio(double visits_num, double visits_denom, double explore_const)
+cdef double _get_ccqnode_scalar_cost(VNode node, Action action)