%&LaTeX%AverFDP.tex by M.Yasuda 1994.9.27 %1994.10.13 revised by Y.Yoshida%AVFD3.tex 1994.12.11 revised by Y.Yoshida %AVFD3.tex 1996.11.26 revised by M.Yasuda %AVFD32.tex 1997.02.21 revised by M.Yasuda %   EJOR submitted  1997.02.24%\topmargin = 0cm%\oddsidemargin = 0cm%\evensidemargin = 0cm%\textheight = 23cm%\textwidth = 16cm%\font\bit=cmbxti10 scaled 1200 %%%-------------Definition--------------\newcommand{\ba}{\tilde{a}}\newcommand{\br}{\tilde{r}}\newcommand{\bp}{\tilde{p}}\newcommand{\bq}{\tilde{q}}\newcommand{\bs}{\tilde{s}}\newcommand{\bt}{\tilde{t}}\newcommand{\bu}{\tilde{u}}\newcommand{\bv}{\tilde{v}}\newcommand{\bh}{\tilde{h}}\newcommand{\bn}{\tilde{n}}\newcommand{\bm}{\tilde{m}}\newcommand{\Tn}{\tilde{n}}\newcommand{\Tm}{\tilde{m}}\newcommand{\Tu}{\tilde{u}}\newcommand{\Tv}{\tilde{v}}\newcommand{\Th}{\tilde{h}}\newcommand{\tf}{\tilde{f}}\newcommand{\al}{\alpha}\newcommand{\cF}{{\cal F}}  % use {{ }} instead of { }\newcommand{\cB}{{\cal B}}  % use {{ }} instead of { }\newcommand{\cA}{{\cal A}}  % use {{ }} instead of { }\newcommand{\cC}{{\cal C}}  % use {{ }} instead of { }\newcommand{\bg}{\tilde{g}}\newcommand{\bR}{\tilde{R}}\newcommand{\veps}{\varepsilon}\newcommand{\af}{\alpha}%\newcommand{\qed}{\hfill {\it q.e.d.}}\newcommand{\qed}{\quad $\Box$}\newcommand{\ovlim}{\overline{\lim}}\newcommand{\udlim}{\underline{\lim}}\def\dfrac#1#2{{\displaystyle\frac{#1}{#2}}}%%%---------------------------\documentstyle[12pt]{article}\begin{document}\def\picture #1 by #2 (#3){  \vbox to    #2{      \hrule width #1 height 0pt depth 0pt      \vfill      \special{picture #3}      }  }\def\scaledpicture #1 by #2 (#3 scaled #4){{  \dimen0=#1 \dimen1=#2  \divide\dimen0 by 1000 \multiply\dimen0 by #4  \divide\dimen1 by 1000 \multiply\dimen1 by #4  \picture \dimen0 by \dimen1 (#3 scaled #4)}  }\def\Average{\scaledpicture 3.92in by 3.39in (Average scaled 800)}\def\UpValue{\scaledpicture 3.92in by 3.39in (UpValue scaled 800)}\def\LoValue{\scaledpicture 3.92in by 3.39in (LoValue scaled 800)}%------------------LaTeX-------------------------------\begin{center}\begin{Large}{\bf Fuzzy Decision Processes with an Average Reward Criterion}%(Kurano Note '94)\end{Large}\end{center}\begin{small}$\mbox{Masami KURANO}^{\dag}, \>\mbox{Masami YASUDA}^{\ddag}, \>\mbox{Jun-ichi NAKAGAMI}^{\ddag} \>$ $  \mbox{ and } \>\mbox{Yuji YOSHIDA}^{\S}$\vspace{2ex}{\it${}^{\dag}${Faculty of Education,}${}^{\ddag}${Faculty of Science, Chiba University,}Chiba 263-8522, Japan. ${}^{\S}${Faculty of Economics and Business Administration, Kitakyushu University,}Kitakyushu 802-8577, Japan.}\end{small}%\date{}%\maketitle%\title{Fuzzy Decision Processes with an Average Reward Criterion}%\author{Kurano,M., etc}\begin{abstract}As the same framework of Fuzzy decision processeswith the discounted case we will specify anaverage fuzzy criterion model and develop itsoptimization by  ``fuzzy max order"  under appropriate conditions. The average reward is characterized, by introducing a relative value function, as a unique solution of the associated equation. Also we derive the optimality equation using the ``vanishing discount factor" approach.\end{abstract}\noindent{\bf Keywords:}  Fuzzy sets; Dynamicprogramming; Decision theory; Averagereward criterion; Relative value function; Optimalityequation.% --- section  1 ----\section{Introduction and notations}In the previous paper~\cite{KurEt2},  Markov-type fuzzy decision processes (FDP's, for short) with a bounded fuzzy reward are defined.We have developed its optimization under the discount reward criterion.Also, the long-run average reward of  somedynamic fuzzy system has been specified in ouranother paper\cite{KurEt3}.  However, the optimization wasnot considered there. In this paper,we will specify the long-run average fuzzy reward from a fuzzy policy and develop its optimization by the so-called ``fuzzy max order" on the convex fuzzynumbers under the ergodicity (contraction) condition for the fuzzy state transition and the continuity condition for the fuzzy reward relation.By introducing the relative value functions, the average reward from any admissible stationary policy is characterized as a unique solution of the associated equation. It will be useful in the policy improvement.Moreover, using the ``vanishing discount factor" approach which is well-known in the theory of Markov decision processes (for example, see \cite{Ros,Tay}), we derive the optimality equation under the average fuzzy reward criterion.In the reminder of this section, we will give notations and some mathematical facts.%%%Let $E$, $E_{1}$, $E_{2}$ be convex compact subsets of a given Banach space.Throughout the paper we will denote a fuzzy setand a fuzzy relation by their membership functions.For the theory of fuzzy sets,refer to Zadeh~\cite{Zad} andNov\^^ {a}k~\cite{Nov}. A fuzzy set $\Tu : E \to [0,1]$ on $E$ is called convex if$$  \Tu( \lambda x + (1- \lambda ) y ) \ge \Tu(x) \wedge \Tu(y),  \quad x,y \in E, \ \lambda \in [0,1],$$where $a \wedge b := \min\{ a,b\}$ (c.f.\cite{Che,Yas1}). Also, a fuzzyrelation$\Th : E_{1} \times E_{2} \to [0,1]$between $E_{1}$ and $E_{2}$is called convex if$$  \Th( \lambda x_{1} + (1- \lambda ) x_{2},  \lambda y_{1} + (1- \lambda ) y_{2}) \ge  \Th(x_{1},y_{1}) \wedge \Th(x_{2},y_{2})$$for $x_{1},x_{2} \in E_1$, \ $y_{1},y_{2} \in E_2$and $\lambda \in [0,1]$.The $\alpha$-cut ($\alpha \in [0,1]$)of the fuzzy set $\Tu$ is defined by$$  {\Tu}_{\alpha} :=  \{ x \in E \mid \Tu(x) \geq \alpha \}   \ (\alpha > 0) \quad \mbox{and} \quad  \Tu_{0} := \mbox{{\it cl} } \{ x \in E \mid \Tu(x) > 0 \},$$where {\it cl} denotes the closure of a set.%Also, we denote the level set of $\Tu$ by %$$%\Tu_{=\al} := \{ x \in E \mid \Tu(x)=\al \}.%$$Let ${\cal F}(E)$ be the set of all convex fuzzy sets, $\Tu$, on $E$whose membership functions are upper semi-continuousand have compact supportsand the normality condition : $\displaystyle{\sup_{x \in E} \Tu (x) = 1}$.We denote by ${\cal C}(E)$the collection of all compact convex subsets of $E$and $\rho_{E}$ the Hausdorff metric on ${\cal C}(E)$.Clearly, $\Tu \in {\cal F}(E)$means $\Tu_{\alpha} \in {\cal C}(E)$for all $\alpha \in [0,1]$.Let ${\bf R}$ be the set of real numbers.We see from the definition that${\cal C}({\bf R})$and${\cal F}({\bf R})$are the set of all bounded closed intervals in ${\bf R}$and all upper semi-continuous and convex fuzzy numbers on ${\bf R}$with compact supports, respectively.For the interval $[0,M]$ with a fixed positive number $M$,$$  {\cal F}([0,M]) = \{ \bu \in {\cal F}(R) \mid                           \bu_0 \subset [0,M] \},$$and ${\cal C}([0, M])$ be the set of all closed convex subsets of $[0, M]$.For non-empty closed intervals, the Hausdorff metric on ${\cal C}([0, M])$ is represented by $\delta$, i.e.,$$  \delta([a,b],[c,d]) := \mid a - c \mid \vee \mid b - d \mid  \quad \mbox{for} \quad [a,b],[c,d] \in {\cal C}([0,M]),$$where $a \vee b = \max\{ a,b\}$ for real numbers $a,b$.%Let %$\hat{\cal F}(\bf R)$ %be the set of fuzzy sets whose level sets %are bounded and closed intervals.%Clearly, %$\hat{\cal F}(\bf R) \supset %{\cal F}(\bf R)$.  Since any fuzzy set%belonging to %$\hat{\cal F}(\bf R)$ %does not necessarily satisfy the normality%condition,  the addition on %$\hat{\cal F}(\bf R)$ %are defined through %its level set as follows :  The definitions of addition and scalar multiplication on ${\cal F}(\bf R)$ are as follows:For $\Tm, \Tn \in {\cal F}(R)$and $\lambda \geq 0$, $$     (\Tm + \Tn)(x) :=     \sup_{x_{1},x_{2} \in {\bf R} : \ x_{1}+x_{2}=x}     \{\Tm(x_{1}) \wedge \Tn(x_{2})\}$$and, for the scalar multiplication, let us define$$  ( \lambda {\hat m})(x) :=     \left\{       \begin{array}{ll}        {\hat m}(x/\lambda) & \mbox{if} \ \lambda>0 \\        I_{\{0\}}(x) & \mbox{if} \ \lambda=0,       \end{array}       \quad (x \in {\bf R})     \right.$$where $I_{\{\cdot\}}(\cdot)$ is an indicator.By using the set operation  $A + B := \{ x + y \mid x \in A, y \in B \}$,\$A + \emptyset = A$for any non-empty sets $A, B \subset {\bf R}$,and $\lambda A := \{ \lambda x \mid x \in A \}$ for $ A\subset {\bf R}$, the following holds immediately.$$    (\Tm+\Tn)_\alpha = \Tm_\alpha +\Tn_\alpha      \quad \mbox{and} \quad    (\lambda\Tm)_\alpha = \lambda \Tm_\alpha \ (\al \in [0,1]) $$ \noindent{\bf Lemma 1.1} \ (\cite[Theorem 2.3]{Che}).\begin{itemize}  \item[(i)]    For any $\bn, \bm \in {\cal F}(R)$ and $\lambda \in R$,    $\bn + \bm \in {\cal F}(R)$ and $\lambda \bn \in {\cal F}(R)$.  \item[(ii)]    For any $\bs \in {\cal F}(E_{1})$ and    $\bp \in {\cal F}(E_{1} \times E_{2})$,    then    $\sup_{x \in E_{1}} \bs(x) \wedge \bp(x,\cdot)    \in {\cal F}(E_{2})$.\end{itemize}%%%In the next section 2, we give the basic definitionof FDP's  and describe the optimization problemby specifying the average fuzzy reward fromany fuzzy policy.In Section 3,  referring \cite{KurEt2} wegive several results for the discounted case which will be also used to derive the optimalityequation by the ``vanishing discount factor''method in Section 5. In Section 4, theaverage fuzzy reward from any stationaryfuzzy policy satisfying some reasonablecondition  is characterized as a uniquesolution of the associated relationalequation. The numerical example is given toillustrate the theoretical results.% ---- section 2 ----\section{Fuzzy Decision Processes}In this section, we will formulate FDP's. By introducing the fuzzy max order on the convex fuzzy numbers the optimization problem could be described.A fuzzy decision process, in this paper, is a controlled dynamic fuzzy system defined by four objects $(S, A, \bq, \br)$ as follows:\begin{itemize}  \item[(i)]     Let $S$ and $A$ be a state space and an action space,     which are given as convex compact subsets of some     Banach space respectively.     The decision process is assumed to be fuzzy itself, so that both the state     of the system and the action taken at each stage are denoted by the     element of ${\cal F}(S)$ and ${\cal F}(A)$,     called the fuzzy state and the fuzzy action respectively.  \item[(ii)]     The law of motion for the system and the fuzzy reward can be     characterized by time invariant fuzzy relations     $\bq \in {\cal F}(S \times A \times S)$ and     $\br \in {\cal F}(S \times A \times [0,M])$,     where $M$ is a given positive number.     Explicitly, if the system is in a fuzzy state     $\bs \in {\cal F}(S)$ and the fuzzy action $\ba \in{\cal F}(A)$     is chosen, then it transfers to a new fuzzy state $Q(\bs, \ba)$ and     a fuzzy reward $R(\bs, \ba)$ is earned,     where $Q, R$ are defined     by the following:    %$\bq$ : continuous convex fuzzy relation.    %$\bq(x,a,y)\in{\cal F}(S \times A \times S)$ means a    %fuzzy transition,$$   Q(\bs, \ba)(y) := \sup_{(x,a) \in S \times A} \bs(x) \wedge \ba(a) \wedge   \bq(x,a,y) \quad (y \in S)   %\quad \mbox{for} \ \bs \in {\cal F}(S),   \eqno(2.1)$$and$$   R(\bs,\ba)(u) := \sup_{(x,a) \in S \times A} \bs(x) \wedge \ba(a) \wedge   \br(x,a,u) \quad (0 \le u \le M).   \eqno(2.2)$$\end{itemize}\noindentNote that, by Lemma 1.1, it holds that$Q(\bs, \ba)(\cdot) \in {\cal F}(S)$ and$R(\bs, \ba)(\cdot) \in {\cal F}([0,M])$ for all$\bs \in {\cal F}(S), \ba \in {\cal F}(A)$.Firstly we will define a policy based on the fuzzy state and fuzzy actionas follows.Let$\Pi := \{ \pi \vert \pi :  {\cal F}(S) \mapsto {\calF}(A)\}$ be the set of all maps from ${\cal F}(S)$ to${\cal F}(A)$. Any element $\pi \in \Pi$ is called astrategy. A policy, $\check{\pi} = (\pi_1, \pi_2, \pi_3,\cdots)$,  is a sequence of strategies such that $\pi_t \in\Pi$ for each $t$. Especially, the policy $(\pi, \pi, \pi,\cdots)$ is a stationary policy and is denoted by${\pi}^{\infty}$.A fuzzy strategy $\pi \in \Pi$ is called admissible if the $\af$-cut$\pi(\bs)_\af$ of $\pi$ depends only on the scalar $\af$ and the sets$\bs_\af$, that is, it would be written as$$    \pi(\bs)_\af = \pi(\af, \bs_\af) \quad \mbox{for} \    \bs \in {\cal F}(S).  \eqno(2.3)$$For the admissible fuzzy strategy $\pi = \{\pi(\cdot,\cdot)\}$, if$\pi(\af,D)$ is continuous in $(\af, D) \in [0,1]\times {\cal C}(S)$,$\pi$ is called continuous. We denote by $\Pi_A$ and $\Pi_{C}$,respectively, the collections of all admissible and continuous fuzzy strategies.A policy $\check{\pi} = (\pi_1, \pi_2, \cdots)$ is called admissible(continuous resp.) if $\pi_t \in \Pi_A \ (\Pi_{C})$ for all $t \ge 0$.To specify a performance index (or an objective function) and describe an optimal decision problem, let us consider the convergence of a sequence of fuzzy numbers and a partial order.\vspace{2ex}\noindent{\bf Definition 2.1} (c.f. \cite{KurEt1,Nan}). \ For $\bu_t, \bu \in {\cal F}(E)$,$$    \lim_{t \to \infty} \bu_t = \bu$$iff $\lim_{t \to \infty} \sup_{\af \in [0,1]} \rho(\bu_{t,\af},\bu_\af) = 0,$ where $\bu_{t,\af}$ and $\bu_{\af}$ are respectivelythe $\af$-cut of $\bu_t$ and $\bu$.\vspace{4ex}For any closed interval $D \in {\cal C}([0,M])$, we put$D = [\underline{D}, \overline{D}]$, where $\underline{D}$ and$\overline{D}$ are the left and right end points of $D$ respectively.The partial order $\preceq$ on ${\cal C}([0,M])$ is defined as follows :For any $D_1, D_2 \in {\cal C}([0,M])$. $D_1 \succeq D_2$ means that$\overline{D_1} \ge \overline{D_2}$ and $\underline{D_1} \ge \underline{D_2}$.Then,  $({\cal C}([0,M]), \succeq)$ becomes a complete lattice (see\cite{Bir}) and the following lemma holds obviously.\vspace{2ex}\noindent{\bf Lemma 2.1.} \ {\slFor any sequence $\{D_n\}_{n=1}^{\infty}\subset {\cal C}([0,M])$, it holds that\begin{itemize}  \item[{\rm (i)}]      $\sup_{n \ge 1} D_n = [ \sup_{n \ge 1} \underline{D_n},    \sup_{n \ge 1} \overline{D_n}]$, \quad and  \item[{\rm (ii)}]      if $\sum_{n \ge 1}^{\infty} D_n $ converges,    $\sum_{n \ge 1}^{\infty} D_n = [ \sum_{n \ge 1}^{\infty} \underline{D_n},    \sum_{n \ge 1}^{\infty} \overline{D_n}]$.\end{itemize}}\vspace{2ex}Using the order on ${\cal C}([0,M])$, let us define the partial order $\succeq$on ${\cal F}([0,M])$ which is called a max fuzzy order.\vspace{2ex}\noindent{\bf Definition 2.2.} \ For any $\bn, \bm \in {\calF}([0,M])$,$\bn \succeq  \bm$ iff $\bn_\af \succeq  \bm_\af$ for all $\af \in [0,1]$.\vspace{4ex}For any admissible policy $\check{\pi} = (\pi_1, \pi_2, \cdots)$ and an initial fuzzy state $\bs \in {\cal F}(S)$, we can define a sequence of fuzzy rewards on $[0,M]$,$$   \{R(\bs_t, \pi_t(\bs_t))\}_{t=1}^{\infty},$$where$$    \bs_1 = \bs \quad \mbox{and} \quad \bs_{t+1} = Q(\bs_t,\pi_t(\bs_t)) \ \mbox{for} \ t \ge 1.    \eqno(2.4)$$\vspace{2ex}Here we are concerned with two performance criteria. The first one is the total discounted fuzzy reward with a discount factor $\beta \ (0 < \beta <1)$;$$    \psi_{\beta}(\check{\pi}, \bs) := \sum_{t=1}^{\infty} \beta^{t-1}    R(\bs_t,\pi_t(\bs_t))     \in {\cal F}([0,M/(1-\beta)])    \eqno(2.5)$$for $\bs \in {\cal F}(S)$ and $\check{\pi} = (\pi_1, \pi_2,\cdots)$.The problem in the discounted case is to maximize$\psi_{\beta}(\check{\pi},\bs)$ over all admissible policy $\check{\pi}$ with respect to the order $\succeq$ on ${\cal F}([0,M])$, which has been investigated in \cite{KurEt2}.The second performance criteria is the long-run average fuzzy reward per unit time, which is formally defined by$$   \Psi(\bs,\check{\pi}) := \lim_{T \to \infty} \frac{R_T}{T},   \eqno(2.6)$$where$$   R_T := \sum_{t=1}^{T} R(\bs_t, \pi_t(\bs_t)) \quad (T \ge 1).   \eqno(2.7)$$Our problem in this paper is to show the convergence property of $\Psi(\bs,\check{\pi})$ and maximize $\Psi(\bs, \check{\pi})$ over some class of continuous policies $\check{\pi}$ with respect to the order $\succeq$ on ${\cal F}([0,M])$,which is given in Sections 4 and 5.% ---- section 3 ----\section{Assumptions and preliminary results}In this section, we introduce some assumptions for the fuzzy relation $\bq$ and fuzzy reward $\br$. And several preliminary results are given for the discounted case, which guarantee the validity of the ``vanishing discount factor" approach.In order to discuss the structure of the fuzzy state transition and fuzzy reward, let us introduce some notations.A map $Q_\af: {\cal C}(S) \times {\cal C}(A) \mapsto {\cal C}(S) \ (\af \in [0,1])$ is defined by$$   Q_\af (D \times B) :=   \left\{     \begin{array}{ll}        \{y \in S \mid \bq(x,a,y) \ge \af \ \mbox{for some} \ (x,a) \in D\times B \},          &    \af > 0, \\        \mbox{{\it cl}} \{y \in S \mid \bq(x,a,y) > 0 \ \mbox{for some} \ (x,a) \in          D\times B \},          &    \af = 0,     \end{array}     \right.$$and a map $R_\af : {\cal C}(S) \times {\cal C}(A) \mapsto {\cal C}([0,M]) \ (\af \in [0,1])$ by$$   R_\af (D \times B) :=   \left\{      \begin{array}{ll}       \{u \in R_{+} \mid \br(x,a,u) \ge \af         \mbox{ for some }            (x,a) \in D\times B \},         &   \af > 0,  \\       \mbox{{\it cl}} \{u \in R_{+} \mid \br(x,a,u) > 0           \mbox{ for some } (x,a)         \in D\times B \},          &   \af = 0.      \end{array}     \right.$$Since $R_\af(D\times B)$ is a closed interval for each $\af \in [0,1]$, we can write it as$$  R_\af (D \times B) := [\underline{R}_{\af} (D \times B), \overline{R}_{\af} (D \times B)].$$In this paper, we need the following two assumptions.\vspace{2ex}\noindent{\bf Assumption A} (Ergodicity or contraction). \ There exits $\gamma_1 \ (0 < \gamma _1 < 1)$ such that$$   \rho(Q_\af(D\times B), Q_\af(D'\times B)) \le \gamma_1\rho(D,D')   \quad \mbox{for all} \ B \in {\cal C}(A).$$\vspace{2ex}\noindent{\bf Assumption B} (Lipschitz condition).  \ There exists a constant $C$ such that$$  \vert \underline{R}_{\af}(D\times B) -  \underline{R}_{\af}(D'\times B) \vert \vee  \vert \overline{R}_{\af}(D\times B) -  \overline{R}_{\af}(D'\times B) \vert    \le C \rho(D, D')$$%$$%  \begin{array}{ll}%      & \vert \underline{R}_{\af}(D\times B) -%          \underline{R}_{\af}(D'\times B')\vert \vee%          \vert \overline{R}_{\af}(D\times B) -%          \overline{R}_{\af}(D'\times B')\vert  \\%      & \le    c \rho(D, D')%  \end{array}% $$for any  $D,D' \in {\cal C}(S)$ and $B \in {\cal C}(A)$.\vspace{2em}We note that Assumption A has been given in\cite{KurEt1, YosEt1},  under whichseveral limit theorems for the sequence offuzzy states have bee obtained.We now derive the optimality equation for the discounted case.Let$$  V := \{ v : {\cal C}(S) \mapsto {\cal C}([0,M]) \}.$$Define a metric $d_V$ on $V$ by$$  d_V(v, w) := \sup_{D \in {\cal C}(S)} \delta(v(D),w(D))  \quad \mbox{for} \  v, w \in V.$$Then, $(V, d_V)$ becomes a complete metric space. Define a map $U_{\af}^{\beta} : V \mapsto V \  (\af \ge 0)$ by$$  U_{\af}^{\beta}v(D) := \sup_{B \in {\cal C}(A)} \{R_\af(D\times B) + \beta v(Q_\af(D\times B))\}\eqno(3.1)$$for $v \in V$ and $D \in {\cal C}(S)$,where $\beta$ is a discount factor and $0 < \beta < 1$.If we write $v(D)$ and $U_{\af}^{\beta}v(D)$ respectively by$   v(D) = [\underline{v}(D), \overline{v}(D)]$and$   U_{\af}^{\beta}v(D) = [\underline{U}_{\af}^{\beta}v(D),   \overline{U}_{\af}^{\beta}v(D)],$(3.1) becomes, from Lemma 2.1,$$  \underline{U}_{\af}^{\beta} v(D) =  \sup_{B \in {\cal C}(A)} \{\underline{R}_\af(D\times B)  + \beta  \underline{v}(Q_\af(D\times B))\},  \eqno(3.2)$$$$  \overline{U}_{\af}^{\beta}v(D) =  \sup_{B \in {\cal C}(A)} \{\overline{R}_\af(D\times B)  + \beta \overline{v}(Q_\af(D\times B))\}.  \eqno(3.3)$$In \cite{KurEt2}, it is shown that the operator$U_{\af}^{\beta}$ is a contraction with modulus $\beta$.Thus, there exists a unique map $v_{\af, \beta} \in V$such that$$ v_{\af,\beta} = U_{\af}^{\beta}v_{\af,\beta}. \eqno(3.4)$$Let $v_{\af,\beta}(D) := [\underline{v}_{\af,\beta}(D),\overline{v}_{\af,\beta}(D)]$ for all $D \in {\cal C}(S)$.The property of $v_{\af,\beta}$ is given in the following lemma.\vspace{2ex}\noindent{\bf Lemma 3.1.}{\sl Suppose that Assumptions A and B hold. Then, we have$$  \vert \underline{v}_{\af,\beta}(D) -\underline{v}_{\af,\beta}(D') \vert  \vee \vert \overline{v}_{\af,\beta}(D) -\overline{v}_{\af,\beta}(D') \vert  \le \frac{C}{1-\beta \gamma_1} \rho(D,D')   \eqno(3.5)$$for all $D, D' \in {\cal C}(S)$.}\vspace{2ex}\noindent{\it Proof.}Putting $\underline{v}_{\af,\beta}^{0} \equiv \{0\} \in V$, we define the iterates $\underline{v}_{\af,\beta}^{t} $ by$$  \underline{v}_{\af,\beta}^{t+1}  = \underline{U}_{\af}^{\beta}  \underline{v}_{\af,\beta}^{t}, \quad (t \ge 0)  \eqno(3.6)$$Then, by the contractive property of$\underline{U}_{\af}^{\beta}$,$\underline{v}_{\af,\beta}^{t}(D) \to \underline{v}_{\af,\beta}(D)$ uniformly for $D \in {\cal C}(S)$ as $ n \to \infty$.Now, we show by induction on $t$ that$$ \vert  \underline{v}_{\af,\beta}^{t}(D)  -  \underline{v}_{\af,\beta}^{t} (D') \vert \le  C \sum_{l=0}^{t-1}(\beta \gamma_1)^l \rho(D,D')  \eqno(3.7)$$for all $D, D' \in {\cal C}(S)$ and $t \ge 1$.It holds, from Assumption B, that$$\vert  \underline{v}_{\af,\beta}^{1}(D)  -  \underline{v}_{\af,\beta}^{1} (D') \vert \le  \sup_{B \in {\cal C}(A)}  \vert  \underline{R}_{\af}(D\times B)  -  \underline{R}_{\af}(D'\times B) \vert  \le C \rho(D,D'),$$which implies that (3.7) holds for $t=1$.Suppose that (3.7) holds for $t$. Then, we have\begin{eqnarray*}  \lefteqn{ \vert \underline{v}_{\af,\beta}^{t+1}(D)    - \underline{v}_{\af,\beta}^{t+1} (D') \vert } \\  & \le &  \sup_{B \in {\cal C}(A)}  \vert  \underline{R}_{\af}(D \times B)  -  \underline{R}_{\af}(D'\times B) \vert \\  & + &  \beta \sup_{B \in {\cal C}(A)}  \vert  \underline{v}_{\af,\beta}^{t} (Q_\af(D\times B))  -  \underline{v}_{\af,\beta}^{t} (Q_\af(D'\times B)) \vert \\  & \le &  C \rho(D,D')   + \beta \sum_{l=0}^{t-1}(\gamma_1 \beta)^{l} \sup_{B\in {\cal C}(A)}  \rho(Q_\af(D \times B), Q_\af(D'\times B)) \\  & & \mbox{\qquad \qquad \qquad, from Assumption B and the hypothesis of induction} \\  & \le & \sum_{l=0}^{t}(\gamma_1\beta)^l \rho(D,D') \\  & & \mbox{\qquad \qquad \qquad, from Assumption A.}\end{eqnarray*}This  shows that (3.6)  holds for $t+1$.  Letting $t \to \infty$ in (3.7),  we get$$  \vert  \underline{v}_{\af,\beta}(D)  -  \underline{v}_{\af,\beta}(D') \vert  \le \frac{C}{1- \beta\gamma_1} \rho(D,D').  \eqno(3.8)$$Similarly as the case of $\underline{v}_{\af,\beta}$,  we obtain that$$  \vert  \overline{v}_{\af,\beta}(D)  -  \overline{v}_{\af,\beta}(D') \vert  \le \frac{C}{1- \beta\gamma_1} \rho(D,D').  \eqno(3.9)$$These inequalities (3.8) and (3.9) implies (3.5).\qed%----- section 4 ----\section{Characterization of the average fuzzy reward}This section concerns the convergence and characterization of the average fuzzy reward $\Psi(\bs,\pi^{\infty})$,which formally given in (2.6).\vspace{2ex}Let $\pi \in \Pi_{C}$.For simplicity, we put$$R_{T}(\bs,\pi^{\infty}) := \sum_{t=1}^{T} R(\bs_{t},\pi(\bs_{t})),\eqno(4.1)$$where $$\bs_{1} := \bs \quad \mbox{and} \quad\bs_{t+1} = Q(\bs_{t},\pi(\bs_{t})) \ (t \ge 1)$$By using $\bq$ and $\br$, we define maps $Q^{\pi}_{\af} : {\cal C}(S) \mapsto {\cal C}(S)$and$R^{\pi}_{\af} : {\cal C}(S) \mapsto {\cal C}([0,M])   \ (\pi \in \Pi_{A}, \af \in [0,1])$ by$$\begin{array}{ll}  Q^{\pi}_{\af}(D) & :=     Q_{\af}(D \times \pi(\af,D)) \\  R^{\pi}_{\af}(D) & :=     R_{\af}(D \times \pi(\af,D)) \end{array}$$for $D \in {\cal C}(S)$.For any admissible fuzzy policy $\pi \in \Pi_{A}$,$Q^{\pi}_{t,\af}$ ($t \ge 1$) is defined inductively by using the composition of maps as follows :$$  Q^{\pi}_{1,\af}(D) := Q^{\pi}_{\af}(D)$$and$$  Q^{\pi}_{t+1,\af}(D)    := Q^{\pi}_{t,\af} Q^{\pi}_{\af}(D)$$for $t \ge 1$ and $ D \in {\cal C}(S)$.\vspace{4ex}The following lemma can be proved analogously to those of \cite[ Lemma 2.1]{KurEt3}.\vspace{2ex}\noindent{\bf Lemma 4.1.}{\sl Let $\pi \in \Pi_{C}$. Then :\begin{itemize}\item[{\rm (i)}]$\bs_{t+1,\af} = Q_{t,\af}^{\pi}(\bs_{\af})$ \ for $t \ge 1$,\item[{\rm (ii)}]$R_{T}(\bs,\pi^{\infty}) \in {\cal F}([0,TM])$ \ for $T \ge 1$,\item[{\rm (iii)}]$(R_{T}(\bs,\pi^{\infty}))_{\af} = \sum_{t=1}^{T} R_{\af}(\bs_{t,\af},\pi(\af,\bs_{t,\af}))$ \ for $T \ge 1$.\end{itemize}}\vspace{2ex}In order to insure the ergodicity of the processand the uniform continuity of the fuzzy reward, we introduce the condition $L(\pi)$ depending on $\pi \in \Pi_{C}$.For any continuous strategy $\pi \in \Pi_{C}$,we shall say that $L(\pi)$ holds if there exist constant $\gamma \, (0 < \gamma <1)$,$C > 0$ and a positive integer $t_{0}$satisfying the following (i) and (ii) :\begin{itemize}\item[(i)]$\rho( Q_{t_{0},\af}^{\pi}(D), Q_{t_{0},\af}^{\pi}(D')) \le \gamma \rho(D,D')$,\item[(ii)]$\delta( R_{\af}^{\pi}(D), R_{\af}^{\pi}(D')) \le C \rho(D,D')$.\end{itemize} \noindentNote that the conditions (i) and (ii) are corresponding to Assumption A and B in Section 3 respectively. Let us denote by $\Pi_{SC}^{*}$ the set of all continuous strategies $\pi \in \Pi_{C}$ satisfying the above condition $L(\pi)$.The convergence of $\Psi(\bs,\pi^{\infty})$ is given in the following theorem.The proof is analogous to those of \cite{KurEt3}.\vspace{2ex}\noindent{\bf Theorem 4.1.}{\sl For any $\pi \in \Pi_{SC}^{*}$, $\Psi(\bs,\pi^{\infty})$ in (2.6) converges and satisfies the following :$$\Psi(\bs,\pi^{\infty}) = R(\bp^{\pi},\pi(\bp^{\pi})),$$where $\bp^{\pi} \in {\cal F}(S)$ is a limiting fuzzy statesatisfying that\begin{itemize}\item[{\rm (i)}]$\lim_{t \to \infty} \bs_{t} = \bp^{\pi}$, and $Q_{\af}^{\pi}(\bp_{\af}^{\pi}) = \bp_{\af}^{\pi}$ for all $\af \in [0,1]$,\item[{\rm (ii)}]$\bp^{\pi}$ is independent of the initial fuzzy state $\bs$, and\item[{\rm (iii)}]$\rho(\bs_{t,\af},\bp^{\pi}_{\af}) \le C^{*} \gamma^{t}$ \ with $\gamma$ in Assumption $L(\pi)$ and some $C^{*} > 0$.\end{itemize}}\vspace{4ex}Recently, Yoshida \cite{Yos2} has given the notion of $\af$-recurrent set for the fuzzy relation and shown that the $\af$-cut of the limiting fuzzy set $\bp^{\pi}$ in the above lemma is characterized as the maximum $\af$-recurrent set.Theorem 4.1 says that for any $\pi \in \Pi_{SC}^{*}$,$\Psi(\bs,\pi^{\infty})$ is independent of $\bs$, so we write it by $\Psi(\pi^{\infty})$.\vspace{2ex}For simplicity, let, for each $\pi \in \Pi_{SC}^{*}$ and $D \in {\cal C}(S)$,$$R_{T,\af}^{\pi^{\infty}}(D) = \sum_{t=1}^{T} R_{\af}^{\pi}(Q_{t,\af}^{\pi}(D)).$$Note from Lemma 4.1 that $R_{T}(\bs,\pi^{\infty})_{\af} = R_{T,\af}^{\pi^{\infty}}(\bs_{\af})$for all $T \ge 1$ and $\af \in [0,1]$.Let $R_{T,\af}^{\pi^{\infty}}(D) := [ \underline{R}_{T,\af}^{\pi^{\infty}}(D),\overline{R}_{T,\af}^{\pi^{\infty}}(D)]$.Then, by Lemma 2.1, we have$$\underline{R}_{T,\af}^{\pi^{\infty}}(D) = \sum_{t=1}^{T} \underline{R}_{\af}^{\pi}(Q_{t,\af}^{\pi}(D))\eqno(4.2)$$and$$\overline{R}_{T,\af}^{\pi^{\infty}}(D) = \sum_{t=1}^{T} \overline{R}_{\af}^{\pi}(Q_{t,\af}^{\pi}(D)),\eqno(4.3)$$where$$R_{\af}^{\pi}(D') := [ \underline{R}_{\af}^{\pi}(D'),\overline{R}_{\af}^{\pi}(D')]$$for all $D' \in {\cal C}(S)$.By Theorem 4.1 and Assumption B,we observe that $R_{\af}^{\pi}(\bs_{t,\af}) \to R_{\af}^{\pi}(\bp_{\af}^{\pi})$exponentially first as $t \to \infty$.Thus, by (4.2) and (4.3),$$\underline{h}_{\af}^{\pi}(D) := \lim_{T \to \infty}(\underline{R}_{T,\af}^{\pi^{\infty}}(D) - T \times \underline{R}_{\af}^{\pi}(\bp_{\af}^{\pi}))\eqno(4.4)$$and $$\overline{h}_{\af}^{\pi}(D) := \lim_{T \to \infty}(\overline{R}_{T,\af}^{\pi^{\infty}}(D) - T \times \overline{R}_{\af}^{\pi}(\bp_{\af}^{\pi}))\eqno(4.5)$$converge for all $D \in {\cal C}(S)$.The function $\underline{h}_{\af}$ ($\overline{h}_{\af}$ resp.) is called lower (upper) relative value function, whose basic ideas are appearing in the theory of Markov processes (c.f. \cite{Ros}).\vspace{2ex}Let us denote the $\af$-cut of the discounted fuzzy reward :$$\psi_{\beta}(\pi^{\infty},\bs)_{\af} := [ \underline{\psi}_{\beta}(\pi^{\infty},\bs)_{\af},\overline{\psi}_{\beta}(\pi^{\infty},\bs)_{\af} ],\quad \af \in [0,1].\eqno(4.6)$$Then, for any $\pi \in \Pi_{SC}^{*}$, the extremal points $$\Psi(\pi^{\infty})_{\af} := [ \underline{\Psi}_{\af}(\pi^{\infty}),\overline{\Psi}_{\af}(\pi^{\infty}) ]$$are characterized in the following theorem, whose description is popular in the theory of Markov decision processes (cf. \cite{BerShr, Bla1, Ros}).\vspace{2ex}\noindent{\bf Theorem 4.2.} {\slFor any $\pi \in \Pi_{SC}^{*}$, we have$$   \underline{\psi}_{\beta}(\pi^{\infty},\bs)_{\af}    = \underline{\Psi}_{\af}(\pi^{\infty}) /(1-\beta)   + \underline{h}_{\af}^{\pi}(\bs_{\af}) + \underline{\varepsilon}(\beta,\af),   \eqno(4.7)$$$$   \overline{\psi}_{\beta}(\pi^{\infty},\bs)_{\af}    = \overline{\Psi}_{\af}(\pi^{\infty}) /(1-\beta)   + \overline{h}_{\af}^{\pi}(\bs_{\af}) + \overline{\varepsilon}(\beta,\af),   \eqno(4.8)$$where $$\vert \underline{\varepsilon}(\beta,\af) \vert \vee\vert \overline{\varepsilon}(\beta,\af) \vert \to 0$$uniformly for $\af \in[0,1]$ as $\beta \to 1$.}\vspace{2ex}\noindent{\it Proof.}From Lemma 2.1, we have$$\begin{array}{ll}\underline{\psi}_{\beta}(\pi^{\infty},\bs)_{\af} & = \sum_{t=1}^{\infty} \beta^{t-1} \underline{R}_{\af}^{\pi}(Q_{t,\af}^{\pi}(\bs_{\af})) \\& = \sum_{t=1}^{\infty} \beta^{t-1} \underline{\Psi}_{\af}(\pi^{\infty}) + \sum_{t=1}^{\infty} \beta^{t-1}(\underline{R}_{\af}^{\pi}(Q_{t,\af}^{\pi}(\bs_{\af})) -\underline{\Psi}_{\af}(\pi^{\infty})) \\& = \underline{\Psi}_{\af}(\pi^{\infty}) /(1-\beta) + \underline{h}_{\af}^{\pi}(\bs_{\af}) + \underline{\varepsilon}(\beta,\af),\end{array}$$where $\underline{\varepsilon}(\beta, \af) =\sum_{t=1}^{\infty} \beta^{t-1}(\underline{R}_{\af}^{\pi}(Q_{t,\af}^{\pi}(\bs_{\af})) -\underline{\Psi}_{\af}(\pi^{\infty})) - \underline{h}_{\af}^{\pi}(\bs_{\af}).$ \\As$\underline{\Psi}_{\af}(\pi^{\infty}) = \underline{R}_{\af}^{\pi}(\bp_{\af})$,by (4.4) and Abel theorem, it holds that$\underline{\varepsilon}(\beta,\af) \to 0$ uniformlyfor $\af \in [0,1] $ as $\beta \uparrow 1$, which proves (4.7).By arguments similar to the above, we can prove (4.8). This completes the proof.\qed\vspace{2ex}The proof of the following theorem is analogous to that of \cite[Theorem 3.1]{KurEt3}.\vspace{2ex}\noindent{\bf Theorem 4.3.} \ {\slFor any $\pi \in \Pi_{SC}^{*}$, let $\underline{h}_{\af}^{\pi}$ and $\overline{h}_{\af}^{\pi}$ be defined as (4.4) and (4.5).Then, the following equations hold : $$\underline{h}_{\af}^{\pi}(D) + \underline{\Psi}_{\af}(\pi^{\infty})= \underline{R}_{\af}^{\pi}(D) + \underline{h}_{\af}^{\pi}(Q_{\af}^{\pi}(D))\eqno(4.9)$$and $$\overline{h}_{\af}^{\pi}(D) + \overline{\Psi}_{\af}(\pi^{\infty})= \overline{R}_{\af}^{\pi}(D) + \overline{h}_{\af}^{\pi}(Q_{\af}^{\pi}(D))\eqno(4.10)$$for all $D \in {\cal C}(S)$.}\vspace{4ex}The following theorem is useful in policy improvement.\vspace{2ex}\noindent{\bf Theorem 4.4.} \{\slFor any $\pi \in \Pi_{SC}^{*}$, let $\underline{h}_{\af}^{\pi}$ and $\overline{h}_{\af}^{\pi}$ be defined as in (4.4) and (4.5).Let $\pi' \in \Pi_{SC}^{*}$ be such that$$\underline{h}_{\af}^{\pi}(D) + \underline{\Psi}_{\af}(\pi^{\infty})\le \underline{R}_{\af}^{\pi'}(D) + \underline{h}_{\af}^{\pi}(Q_{\af}^{\pi'}(D))\eqno(4.11)$$and $$\overline{h}_{\af}^{\pi}(D) + \overline{\Psi}_{\af}(\pi^{\infty})\le \overline{R}_{\af}^{\pi'}(D) + \overline{h}_{\af}^{\pi}(Q_{\af}^{\pi'}(D))\eqno(4.12)$$for all $D \in {\cal C}(S)$ and $\af \in [0,1]$.Then $\Psi(\pi^{\infty}) \preceq \Psi({\pi'}^{\infty})$.}\vspace{2ex}\noindent{\it Proof.} \ Let $\pi' \in \Pi_{SC}^{*}$ be such that (4.11) and (4.12) hold.Then, we have$$\underline{h}_{\af}^{\pi}(Q_{t,\af}^{\pi'}(\bs_{\af})) + \underline{\Psi}_{\af}(\pi^{\infty})\le \underline{R}_{\af}^{\pi'}(Q_{t,\af}^{\pi'}(\bs_{\af})) + \underline{h}_{\af}^{\pi}(Q_{t+1,\af}^{\pi'}(\bs_{\af}))\quad \mbox{for all} \ t \ge 1.$$Summing up the above inequality for $1 \le t \le T$, we get$$\underline{h}_{\af}^{\pi}(\bs_{\af}) + T \times \underline{\Psi}_{\af}(\pi^{\infty})\le \underline{R}_{T,\af}^{{\pi'}^{\infty}}(\bs_{\af}) + \underline{h}_{\af}^{\pi}(Q_{T+1,\af}^{\pi'}(\bs_{\af}))\quad \mbox{for all} \ t \ge 1.$$Thus, since $$\underline{\Psi}_{\af}({\pi'}^{\infty})= \lim_{T \to \infty} \frac{1}{T} \underline{R}_{T,\af}^{{\pi'}^{\infty}}(\bs_{\af}),$$it holds that$$\underline{\Psi}_{\af}(\pi^{\infty})\le \underline{\Psi}_{\af}({\pi'}^{\infty}).$$Similarly$$\overline{\Psi}_{\af}(\pi^{\infty})\le \overline{\Psi}_{\af}({\pi'}^{\infty}).$$Therefore,$$[ \underline{\Psi}_{\af}(\pi^{\infty}), \overline{\Psi}_{\af}(\pi^{\infty}) ] \preceq [ \underline{\Psi}_{\af}({\pi'}^{\infty}), \overline{\Psi}_{\af}({\pi'}^{\infty}) ],$$which shows$$\Psi(\pi^{\infty})\preceq \Psi({\pi'}^{\infty}).$$Thus we get this theorem.\qed\vspace{2ex}Applying the method of the proof in Theorem 4.4, we can easily prove the following corollary.\vspace{2ex}\noindent{\bf Corollary 4.1.} \{\slFor any $\pi \in \Pi_{SC}^{*}$, let $\underline{h}_{\af}^{\pi}$ and $\overline{h}_{\af}^{\pi}$ be defined as in (4.4) and (4.5).Suppose that the following inequalities hold :$$\underline{h}_{\af}^{\pi}(D) + \underline{\Psi}_{\af}(\pi^{\infty})\ge \underline{R}_{\af}^{\pi'}(D) + \underline{h}_{\af}^{\pi}(Q_{\af}^{\pi'}(D))$$and $$\overline{h}_{\af}^{\pi}(D) + \overline{\Psi}_{\af}(\pi^{\infty})\ge \overline{R}_{\af}^{\pi'}(D) + \overline{h}_{\af}^{\pi}(Q_{\af}^{\pi'}(D))$$for all $\pi' \in \Pi_{SC}^{*}$, $D \in {\cal C}(S)$ and $\af \in [0,1]$.Then $\pi^{\infty}$ is absolutely optimal in $\Pi_{SC}^{*}$, i.e.,$$\Psi(\pi^{\infty}) \succeq \Psi({\pi'}^{\infty})\quad \mbox{for all} \ \pi' \in \Pi_{SC}^{*}.$$}%--------------------- Section 5  -----------------------------\section{The optimality equation}In this section, we derive the optimality equation and consider its validity for optimization.The proof is done by the ``vanishing discounted factor" method,using Ascoli-Arzela theorem (c.f.\cite{FerGau, Kur, Roy}).\vspace{2ex}\noindent{\bf Theorem 5.1.} \{\slSuppose that Assumptions A and B in Section 3 hold.Then, for any $\af \in [0,1]$, there exist constants $\underline{\Psi}_{\af}, \overline{\Psi}_{\af}$ and $\underline{v}_{\af}, \overline{v}_{\af} : {\cal C}(S) \mapsto {\cal C}([0,M])$ such that \begin{itemize}\item[{\rm (i)}]$[\underline{\Psi}_{\af'}, \overline{\Psi}_{\af'}] \supset [\underline{\Psi}_{\af}, \overline{\Psi}_{\af}]$\hfill {\rm (5.1)}  \end{itemize}for any $\af,\af' \, (\af' < \af)$ belonging to some countable subset dense in $[0,1]$,and\begin{itemize}\item[{\rm (ii)}]$\underline{v}_{\af}(D) + \underline{\Psi}_{\af}= \sup_{B \in {\cal C}(A)} \{ \underline{R}_{\af}(D \times B) +\underline{v}_{\af}(Q_{\af}(D \times B)) \}$\hfill {\rm (5.2)}\item[{\rm (iii)}]$\overline{v}_{\af}(D) + \overline{\Psi}_{\af}= \sup_{B \in {\cal C}(A)} \{ \overline{R}_{\af}(D \times B) +\overline{v}_{\af}(Q_{\af}(D \times B)) \}$\hfill {\rm (5.3)}\end{itemize}for all $D \in {\cal C}(S)$ and $\af \in [0,1]$.}\vspace{2ex}\noindent{\it Proof.} \ Select $D_{0} \in {\cal C}(S)$ arbitrary.For $\underline{v}_{\af,\beta}$ in Lemma 3.1, let$$\underline{g}_{\af,\beta}(D) := \underline{v}_{\af,\beta}(D) - \underline{v}_{\af,\beta}(D_{0})\quad \mbox{for} \ D \in {\cal C}(S).$$Then, noting (3.4), it holds that$$ \begin{array}{ll}     & \underline{g}_{\af,\beta}(D) + (1-\beta)    \underline{v}_{\af,\beta}(D_{0}) \\   = & \sup_{B \in {\cal C}(A)}    \{ \underline{R}_{\af}(D \times B) + \beta   \underline{g}_{\af,\beta}(Q_{\af}(D \times B)) \}. \end{array}\eqno(5.4)$$Since $0 \le (1-\beta) \underline{v}_{\af,\beta}(D_{0}) \le M$ for all $\beta \in (0,1)$,there exists a sequence $\{ \beta_{n} \}_{n=0}^{\infty}$ such that $\beta_{n} \uparrow 1$ and $(1-\beta_{n}) \underline{v}_{\af,\beta_{n}}(D_{0}) \to \underline{\Psi}_{\af}$.By Lemma 3.1, $\underline{g}_{\af,\beta_{n}}(\cdot)$, $n \ge 1$, are equi-continuous and uniformly bounded, so that, applying the Ascoli-Arzelatheorem (\cite{Roy}),$\underline{g}_{\af,\beta_{n}}(\cdot) \to \underline{v}_{\af}(\cdot)$uniformly along a subsequence (also called $\{ \beta_{n} \}$ by abuse of notation).Thus, letting $n \to \infty$ in (5.4) for $\beta = \beta_{n}$,we get (5.2).Also, putting $\overline{g}_{\af,\beta}(D) := \overline{v}_{\af,\beta}(D) - \overline{v}_{\af,\beta}(D_{0})$,the proof of (5.3) is given as same as the above.Note that we can assume, without loss ofgenerality, that$(1-\beta_n)\underline{v}_{\af,\beta_n}(D_0)\to \underline{\Psi}_{\af}$and $(1-\beta_n)\overline{v}_{\af,\beta_n}(D_0)\to \overline{\Psi}_{\af}$ for any rationalnumber $\alpha$ in $[0,1]$.Since $$  [ \underline{v}_{\af',\beta}(D_{0}),  \overline{v}_{\af',\beta}(D_{0}) ] \supset  [  \underline{v}_{\af,\beta}(D_{0}),  \overline{v}_{\af,\beta}(D_{0}) ]$$ for all $\af,\af'$ ($\af' <\af$) and $\beta \in [0,1]$ (c.f. \cite{KurEt2}), (i) holdsobviously. This completes the proof.\qed\vspace{4ex}It will be shown in Theorem 5.2 below that $[\underline{\Psi}_{\af}, \overline{\Psi}_{\af}]$given in Theorem 5.1 is corresponding to the $\af$-cut of the maximum average fuzzy reward,so that (5.2) and (5.3) are interpreted as the optimality equations for our average fuzzy decision model.For this purpose, we need the following lemmas.\vspace{2ex}\noindent{\bf Lemma 5.1.} \{\slFor any $\bn, \bm \in {\cal F}([0,M])$,if $\bn_{\af} \succeq (=) \bm_{\af}$ on some subset $F$ dense in $[0,1]$,then $\bn \succeq (=) \bm$.}\vspace{2ex}\noindent{\it Proof.} \ From the denseness of $F$,we observe that for any $\af \in [0,1]$$$\bn_{\af} = \lim_{\af' \uparrow \af \, \mbox{\scriptsize{with}} \, \af' \in F} \bn_{\af'} \succeq (=) \lim_{\af' \uparrow \af \, \mbox{\scriptsize{with}} \, \af' \in F} \bm_{\af'} = \bm_{\af},$$which implies $\bn \succeq (=) \bm$.\qed\vspace{2ex}\noindent{\bf Lemma 5.2} \ (c.f. \cite{KurEt1,Nov}).{\slSuppose that a family of subsets $\{ D_{\af}, \af \in [0,1] \}$ satisfies the following {\rm (i)} and {\rm (ii)} :\begin{itemize}\item[{\rm (i)}]      $D_{\af'} \subset D_{\af}$ for all $\af,\af'  ( 0 \le \af' \le \af \le 1)$,\item[{\rm (ii)}]      $\lim_{\af' \uparrow \af} D_{\af'} = D_{\af}$ for all $\af \in [0,1]$.\end{itemize}Then, $\bs(x) := \sup_{\af \in [0,1]} \{ \af \wedge I_{D_{\af}}(x) \},\ x \in S$, satisfies $\bs \in {\cal F}(S)$and $\bs_{\af} = D_{\af}$ for all $\af \in [0,1]$.}\vspace{4ex}Let denote by $F$ a countable subset on which (i) in Theorem 5.1 holds.For $\underline{\Psi}_{\af}$ and $\overline{\Psi}_{\af}$ given in Theorem 5.1,let, for any $\af \in [0,1]$,$$\Psi_{\af}^{-} := \lim_{\af' \uparrow \af \, \mbox{\scriptsize{with}} \, \af' \in F} \underline{\Psi}_{\af'},\quad \Psi_{0}^{-} := \underline{\Psi}_{0},$$$$\Psi_{\af}^{+} := \lim_{\af' \uparrow \af \, \mbox{\scriptsize{with}} \, \af' \in F} \overline{\Psi}_{\af'},\quad \Psi_{0}^{+} := \overline{\Psi}_{0}.$$Since the conditions (i) and (ii) in Lemma 5.2 hold for the family $\{ [\Psi_{\af}^{-}, \Psi_{\af}^{+}]; \af \in [0,1] \}$,we can construct the fuzzy set $\tilde{\Psi}$ by$$  \tilde{\Psi}(u)  := \sup_{\af \in [0,1]}        \{ \af \wedge 1_{ [ \Psi_{\af}^{-}, \Psi_{\af}^{+}]}(u)\},   \quad u \in [0,\infty).$$\vspace{2ex}Now, we can state the following theorem.\vspace{2ex}\noindent{\bf Theorem 5.2. \ }{\sl\begin{itemize}  \item[{\rm (i)}]$  \tilde{\Psi} \succeq \Psi(\pi^{\infty})  \quad \mbox{for any} \ \pi \in \Pi_{SC}^{*}.$  \item[{\rm (ii)}]If there exists a strategy $\pi^{*} \in \Pi_{SC}^{*}$ such that$$\underline{v}_{\af}(D) + \underline{\Psi}_{\af}= \underline{R}_{\af}(D \times \pi^{*}(\af,D)) + \underline{v}_{\af}(Q_{\af}(D \times \pi^{*}(\af,D)))$$and $$\overline{v}_{\af}(D) + \overline{\Psi}_{\af}= \overline{R}_{\af}(D \times \pi^{*}(\af,D)) + \overline{v}_{\af}(Q_{\af}(D \times \pi^{*}(\af,D)))$$for all $D \in {\cal C}(S)$ and $\af \in [0,1]$,then ${\pi^{*}}^{\infty}$ is absolutely optimal in the family ofcontinuous stationary policies, i.e.,$$  \Psi({\pi^{*}}^{\infty}) \succeq \Psi(\pi^{\infty})  \quad \mbox{for all} \ \pi \in \Pi_{SC}^{*}.$$\end{itemize}}\vspace{2ex}\noindent{\it Proof.} \ Let $\pi \in \Pi_{SC}^{*}$.By the optimality equations (5.2) and (5.3), we have $$\underline{v}_{\af}(D) + \underline{\Psi}_{\af}\ge \underline{R}_{\af}^{\pi}(D)+ \underline{v}_{\af}(Q_{\af}^{\pi}(D))\eqno(5.5)$$and $$\overline{v}_{\af}(D) + \overline{\Psi}_{\af}\ge \overline{R}_{\af}^{\pi}(D)+ \overline{v}_{\af}(Q_{\af}^{\pi}(D))\eqno(5.6)$$for all $D \in {\cal C}(S)$.By the same discussion as that in the proof of Theorem 4.4, we can prove from (5.5) and (5.6) that$$[ \underline{\Psi}_{\af}, \overline{\Psi}_{\af} ]\succeq \Psi(\pi^{\infty})_{\af}\quad \mbox{for all} \ \af \in [0,1].\eqno(5.7)$$Since $[\Psi_{\af}^{-},\Psi_{\af}^{+}] = [ \underline{\Psi}_{\af}, \overline{\Psi}_{\af} ]$ for all $\af \in F$,we obtain (i) of Theorem 5.2 from (5.7) and Lemma 5.1.For (ii), by the same method as the above,we can prove that $\tilde{\Psi} = \Psi({\pi^{*}}^{\infty})$.Thus, together with (i), (ii) follows.This completes the proof.\qed%--------------------- Example  -----------------------------\vspace{4ex}Here we give a numerical example to illustrate the theoretical results in this section.Let $S := [0,1]$, $A := [0,1/2]$ and $M:=1$.Take the fuzzy relation and the fuzzy reward by$$\bq(x,a,y) =( 1 - 3 | y - a x | ) \vee 0, \quad x,y \in [0,1], \ a \in \left[0,1/2 \right]\eqno(5.8)$$and$$\br(x,a,z) = ( 1 - 6 | x - z | ) \vee 0, \quad x,z \in [0,1].\eqno(5.9)$$$\br$ is independent of $a \in [0,1/2]$.Obviously, they satisfy Assumption A for $t_{0}=1$ and Assumption B.Let $\af \in [0,1]$ and $\pi(a) := I_{\{ a \}} \in \Pi_{SC}^{*}$ for $a \in A$.From (5.8) and (5.9), we get$$Q_{\af}^{\pi(a)}(\{ x \}) = \left[\left( a x -  (1-\af)/3  \right) \vee 0,  a x + (1-\af)/3   \right] $$ for $ x \in [0,1], \ a \in \left[0,1/2 \right]$. So$$Q_{\af}^{\pi(a)}([c,b]) = [T_{1}^{a}(c),T_{2}^{a}(b)]\eqno(5.10)$$ for $  0 \le c \le b \le 1$, where maps $T_{i}^{a} : [0,1] \mapsto [0,1]$ ($i=1,2; a \in [0,1/2]$) are given by$$T_{1}^{a}(x) := \left( a x - (1-\af)/3  \right) \vee 0;\quad T_{2}^{a}(x) :=   a x + (1-\af)/3  \quad \mbox{for} \ x \in [0,1].$$Similarly we observe that$$R_{\af}^{\pi(a)}([c,b]) = \left[\left( c - (1-\af)/6  \right) \vee 0,\left( b + (1-\af)/6 \right) \wedge 1 \right]\quad \mbox{for} \ 0 \le c \le b \le 1.$$By (5.10), a unique fixed point $\bp_{\af}^{\pi(a)}$ of the map $Q_{\af}^{\pi(a)} : {\cal C}([0,1]) \mapsto {\cal C}([0,1])$ can be found to be$$\bp_{\af}^{\pi(a)} = \left[ 0, \ (1-\af)/3(1-a) \right].$$Applying Theorem 4.1, we get$$\Psi(\bs,\pi(a)^{\infty})_{\af} = \left[ 0, \ {(1-\af)(3-a)}/{6(1-a)} \right]\quad \mbox{for} \ \af \in [0,1].\eqno(5.11)$$In \cite{KurEt3}, the average fuzzy reward and the relative value functions $\underline{h}_{\af}$ and $\overline{h}_{\af}$for $\pi(1/2)^{\infty}$ have been given as follows :$$\Psi(\bs,\pi(1/2)^{\infty})(x) = \left\{\begin{array}{ll}1 - 6x / 5 & 0 \le x \le 5 / 6 \\0 & 5 / 6 < x \le 1,\end{array}\right.$$%%\begin{center} %\%\Average%\put(2,8){$x$}%\put(-207,150){$\af$}%\end{center}%\begin{center}%Fig. 1 : The average fuzzy reward $\bg(\bs)(x)$.%\end{center}%$$\begin{array}{ll}  & \underline{h}_{\af}^{\pi(1/2)}(c) := \underline{h}_{\af}^{\pi(1/2)}([c,b]) \\ = & \left\{  \begin{array}{ll}    2 \left(1-\left( 1/2 \right)^{t^{*}}\right)      \left( c +  2(1-\af)/3  \right)  -     5 t^{*} (1-\af)/6    & \af < 1 \\   2 c & \af = 1,  \end{array}\right.\end{array}$$where $t^{*}$ is the smallest non-negative integer such that $$\left( {1}/{2} \right)^{t^{*}} \left( c +{2(1-\af)}/{3} \right) - {5(1-\af)}/{6} < 0,$$ and$$\overline{h}_{\af}^{\pi(1/2)}(b) := \overline{h}_{\af}^{\pi(1/2)}([c,b])  = \left\{\begin{array}{ll}2 b -  4(1-\af)/3 & \mbox{if} \ 0 \le b <  (5+\af)/6  \\b +  (3 \af-1)/2  & \mbox{if} \ (5+\af)/6  \le b \le 1.\end{array}\right.$$The graph of $\underline{h}_{\af}^{\pi(1/2)}$ and $\overline{h}_{\af}^{\pi(1/2)}$ for $\af = 1/2$ is given in Figure 1,from which we conjecture the monotonicity of $\underline{h}_{\af}^{\pi(1/2)}$ and $\overline{h}_{\af}^{\pi(1/2)}$.\vspace*{-10ex}\begin{center} \ \UpValue\put(2,45){$x$}%\put(-207,150){$\af$}\put(-50,105){$\overline{h}_{\af}^{\pi(1/2)}(x)$}\put(-50,71){$\underline{h}_{\af}^{\pi(1/2)}(x)$}\end{center}\begin{center}Fig. 1 : The relative value functions $\underline{h}_{\af}^{\pi(1/2)}$ and $\overline{h}_{\af}^{\pi(1/2)}$ for $\af = 1/2$.\end{center}By the definition of Section 3, we see that$$Q_{\af}([c,b] \times [c',b']) = \left[ \left( c c' - {(1-\af)}/{3} \right) \vee0, \left( b b' + {(1+\af)}/{3} \right) \right]$$and$$R_{\af}([c,b] \times [c',b']) = \left[ \left( c - {(1-\af)}/{6} \right) \vee 0, \left( b + {(1-\af)}/{6} \right) \wedge 1 \right]$$for $[c,b] \in {\cal C}(S)$, $[c',b'] \in {\cal C}(A)$.Noting (5.11) for $a = 1/2$, let us put$\underline{\Psi}_{\af}=0$and$\overline{\Psi}_{\af}= 5(1-\af)/6 .$Then, by the monotonicity of $Q_{\af}$, $R_{\af}$, $\underline{h}_{\af}^{\pi(1/2)}$ and $\overline{h}_{\af}^{\pi(1/2)}$,we get the optimality equations described in (5.2) and (5.3) :\begin{eqnarray*}  \qquad \qquad  \lefteqn{ \underline{h}_{\af}^{\pi(1/2)}([c,b]) + \underline{\Psi}_{\af} } \\  & = & \sup_{[c',b'] \in {\cal C}(A)} \{ \underline{R}_{\af}([c,b] \times [c',b'])   + \underline{h}_{\af}^{\pi(1/2)}(Q_{\af}([c,b] \times [c',b'])) \} \\  & = & \underline{R}_{\af}([c,b] \times \{ 1/2 \})   + \underline{h}_{\af}^{\pi(1/2)}(Q_{\af}([c,b] \times \{ 1/2 \}))  \qquad \qquad \mbox{(5.12)}\end{eqnarray*}and \begin{eqnarray*}  \qquad \qquad  \lefteqn{ \overline{h}_{\af}^{\pi(1/2)}([c,b]) + \overline{\Psi}_{\af} } \\  & = & \sup_{[c',b'] \in {\cal C}(A)} \{ \overline{R}_{\af}([c,b] \times [c',b'])   + \overline{h}_{\af}^{\pi(1/2)}(Q_{\af}([c,b] \times [c',b'])) \} \\  & = & \overline{R}_{\af}([c,b] \times \{ 1/2 \})   + \overline{h}_{\af}^{\pi(1/2)}(Q_{\af}([c,b] \times \{ 1/2 \}))  \qquad \qquad \mbox{(5.13)}\end{eqnarray*}for all $[c,b] \in {\cal C}(S)$.From Theorem 5.2, $\pi(1/2)^{\infty}$ becomes absolutely optimal.%--------------------- References  -----------------------------\begin{thebibliography}{99}\baselineskip 1mm\bibitem{BerShr}Bertsekas,D.P. and Shreve,S.E.,{\it Stochastic Optimal Control : the Discrete Time Case} \(Academic Press, New York, 1978).\vspace{-1ex}\bibitem{Bir}Birkhoff,G.,``Lattice Theory", {\it Amer. Math. Soc., Coll. Pub.},{\bf 25} (1940). \vspace{-1ex}\bibitem{Bla1}Blackwell,D.,``Discrete dynamic programming", {\it Ann. Math. Stat.}{\bf 33} (1962) 719-726. \vspace{-1ex}\bibitem{FerGau}Fern\'{a}ndes-Gaucherand,E., ``A note on Ross-Taylor theorem", {\it Applied Math. and Computation}{\bf 64} (1994) 207-212. \vspace{-1ex}\bibitem{KurEt1}Kurano,M., Yasuda,M., Nakagami,J. and Yoshida,Y.,``A limit theorem in some dynamic fuzzy systems", {\it Fuzzy Sets and Systems}{\bf 51} (1992) 83-88.\vspace{-1ex}\bibitem{KurEt2}Kurano,M., Yasuda,M., Nakagami,J. and Yoshida,Y.,``Markov-type fuzzy decision processes with adiscounted reward on a closed interval",  {\it EuropeanJ. of Oper. Res.,} {\bf 92}  (1994) pp.649--662.\vspace{-1ex}\bibitem{KurEt3}Kurano,M., Yasuda,M., Nakagami,J. and Yoshida,Y.,``The time average reward with some dynamic fuzzysystems",  to appear in {\it Comp. Math. Appli}.\vspace{-1ex}\bibitem{Kur}Kuratowski,K., {\it Topology I} \ (Academic Press, New York, 1966). \vspace{-1ex}\bibitem{MadEt}Puri,M.L. and Ralescu,D.A.,``The concept of normality for fuzzy random variables", {\it Ann. Prob.}{\bf 13} (1985) 1373-1379. \vspace{-1ex}\bibitem{Nan}Nanda,S., ``On sequences of fuzzy numbers",{\it Fuzzy Sets and Systems}{\bf 33} (1989) 123-126. \vspace{-1ex}\bibitem{Nov}Nov\^^ {a}k,V., {\it Fuzzy Sets and Their Applications} \ (Adam Hilder, Bristol-Boston, 1989). \vspace{-1ex}\bibitem{Ros}Ross,S.M.,{\it Applied probability models with optimization applications} \(Holden-Say, San Francisco, 1970).\vspace{-1ex}\bibitem{Roy}Royden,H.L., {\it Real Analysis}, \ 2nd ed. (Macmillan, New York, 1968). \vspace{-1ex}\bibitem{Tay}Taylor,H.M., ``Markovian sequential replacement processes", {\it Ann. Math. Statist.}{\bf 38} (1965) 1677-2122. \vspace{-1ex}\bibitem{Che}Xu,Chen-wei, ``On convexity of fuzzy sets and fuzzy relations", {\it Information Science}{\bf 59} (1992) 92-102. \vspace{-1ex}\bibitem{Yas1}Yasuda,M., ``A linear structure and convexity for relations indynamic fuzzy systems",   {\it Computers Math. Applic.}{\bf 27} (1994) 23-27. \vspace{-1ex}\bibitem{YosEt1}Yoshida,Y., Yasuda,M., Nakagami,J. and Kurano,M.,``A potential of fuzzy relations with a linearstructure :  The contractive case",  {\it Fuzzy Sets and Systems}{\bf 60} (1993) 283-294. \vspace{-1ex}\bibitem{YosEt2}Yoshida,Y., Yasuda,M., Nakagami,J. and Kurano,M.,``A potential of fuzzy relations with a linearstructure :  The unbounded case",  {\it Fuzzy Sets and Systems}{\bf 66} (1994) 83-95. \vspace{-1ex}\bibitem{Yos2}Yoshida,Y.,``The recurrence of dynamic fuzzy systems", to appear in {\it Fuzzy Sets and Systems}. \vspace{-1ex}\bibitem{Zad}Zadeh,L.A., ``Fuzzy sets", {\it Inform. and Control}{\bf 8} (1965) 338-353. \end{thebibliography}\end{document}