% latex2e
% FTmdp2005.tex
% FSS special issue vol.127.3
\documentclass{elsart}
\usepackage{amsmath}  % amstex.sty \boldsymbol
\usepackage{amssymb}  % \mathfrak \mathbb
%\usepackage{rsfs}     % \mathscr
\usepackage{latexsym} % \Box
\usepackage{graphicx}
%----------------------Definition------------------------------------
\def\T#1{{\widetilde{#1}}}
\def\C#1{{\mathcal{#1}}}
\def\SC#1{{\mathscr{#1}}}
\def\FR#1{{\mathfrak{#1}}}
\def\BB#1{{\mathbb{#1}}}
\newcommand{\BR}{{\mathbb{R}}}
\def\BS#1{{\boldsymbol{#1}}}
\def\U#1{{\underline{#1}}}
\def\O#1{{\overline{#1}}}
\newcommand{\af}{{\alpha}}
\newcommand{\ld}{{\lambda}}
\newcommand{\inc}{{\preccurlyeq}}
\newcommand{\Pf}{\noindent {\bf Proof.} \ }
%\newcommand{\qed}{{\quad $\square$}}
%%%-------------Definition--------------
\newtheorem{thrm}{Theorem}[section]
\newtheorem{lemm}[thrm]{Lemma}%[section]
\newtheorem{dfnt}[thrm]{Definition}%[section]
\newtheorem{prot}[thrm]{Proposition}%[section]
\newtheorem{corl}[thrm]{Corollary}
\newtheorem{remk}[thrm]{Remark}
\newtheorem{exa}[thrm]{Example}
\newtheorem{nte}[thrm]{Note}
%\numberwithin{equation}{section}
\renewcommand{\theequation}{{\thesection}.\arabic{equation}}
%-------------------------------------------------
\begin{document}
\begin{frontmatter}
%
\title{A fuzzy approach to Markov decision processes 
     with uncertain transition probabilities}
%
\author{M. Kurano\corauthref{cor}}
\address{%Department of Mathematics, 
    Faculty of Education,
    Chiba University,
    Chiba 263-8522 Japan,}
\corauth[cor]{Corresponding author. Tel: +81-43-290-2669}
\ead{kurano@faculty.chiba-u.jp}
%
\author{M. Yasuda}
\address{%Department of Mathematics \& Informatics,
     Faculty of Science, Chiba University, Chiba 263-8522, Japan}
\ead{yasuda@math.s.chiba-u.ac.jp}
%
\author{J.Nakagami}
\address{%Department of Mathematics \& Informatics,
     Faculty of Science, Chiba University, Chiba 263-8522, Japan}
\ead{nakagami@math.s.chiba-u.ac.jp}
%
\author{Y.Yoshida}
\address{Faculty of Economics \& Business Administration,
    University of Kitakyushu,
    Kitakyushu 802-8577 Japan}
\ead{yoshida@kitakyu-u.ac.jp}
%
\begin{abstract}
  In this paper, a Markov decision model with uncertain 
  transition matrices, which allow a matrix to fluctuate 
  at each step in time, is described by the use of fuzzy sets.
  We find a pareto optimal policy maximizing the infinite horizon 
  fuzzy expected discounted reward over all stationary policies 
  under some partial order. 
  The pareto optimal policies are characterized by maximal solutions 
  of an optimal inclusion including efficient set-functions. 
  As a numerical example, the machine maintenance problem is considered.
\end{abstract}
%
\begin{keyword} Fuzzy analysis \sep  Markov decision process \sep
   Pareto optimal \sep Optimal inclusion
\end{keyword}
%
\end{frontmatter}
%---------------------------- Text -----------------------------------
%%%%%%%%%%%%%%%%%%%%%%%%
%      Section 1       %
%%%%%%%%%%%%%%%%%%%%%%%%
\section{Introduction} %---------------------------------------
In a real application of Markov decision processes
(\cite{Bla62,Hin70,How60,MinOsa70,Put94}) (MDP, in short), 
we often encounter the case where the required data is not known 
precisely and perfectly. 
In fact, in many instances, 
the required data in MDPs must be estimated 
through the measurement of various phenomena, 
so that it naturally includes 
imprecision or ambiguity of the observing system. 
Also, it requires to be more {\lq\lq}robust{\rq\rq} 
in the sense that it is reasonably efficient in approximations. 

In order to deal with these uncertain data and flexible requirements, 
Kruce et al.\cite{KBC87} have used a fuzzy set representation 
for homogeneous Markov chains with uncertain transition matrices, 
in which ergodic theorems are obtained in fuzzy environment. 
In this paper, we shall develop a fuzzy treatment for uncertain MDPs 
which allow for fluctuating transition matrices at each step in time. 
The MDPs with uncertain transition matrices are described 
by the use of fuzzy sets, 
in which we fined a Pareto optimal policy maximizing 
the infinite horizon fuzzy expected discounted reward(FEDR) 
over all stationary policies under some partial order relation. 

Associated with each stationary policy 
the corresponding contractive operator is given on fuzzy numbers, 
whose fixed point represents the infinite horizon FEDR.  
Moreover, the Pareto optimal policies are characterized by 
maximal solutions of an optimal inclusion 
including efficient set-functions. 
As a numerical example, the machine maintenance problem is considered. 

Recently, applying Hartfiel's\cite{HarSen94,Har98} interval method 
for Markov chains, Kurano et al.\cite{KSHH98} have introduced 
a decision model, called a controlled Markov set-chain, 
which is robust for rough approximation of transition matrices in MDPs. 
Our fuzzy decision model examined in this paper includes 
a controlled Markov set-chain as a special case. 
So, the results obtained here can be thought of 
as a fuzzy extension of those in \cite{KSHH98}. 
For the optimization of fuzzy dynamic system, 
refer to \cite{KYNY96,Yos98}. 
The non-discounted reward problem for a controlled Markov set-chain 
was developed in \cite{HosKur,KYN99}. 

This paper is organized as follows: 
In Section 2, 
we shall give some notation on fuzzy sets and interval arithmetics 
and obtain the preliminary lemmas. 
In Section 3, 
we describe a nonhomogeneous MDPs by the use of fuzzy sets 
and specify the optimization problem. 
In Section 4, 
the infinite horizon FEDR from a stationary policy is given 
as a fixed point of a corresponding operator, 
which is used to obtain the optimality equation 
and characterize a Pareto optimal policy in Section 5.


%%%%%%%%%%%%%%%%%%%%%%%%
%      Section 2       %
%%%%%%%%%%%%%%%%%%%%%%%%
\section{Notation and preliminary lemmas} %-----------------------------
Let 
$\BR$, $\BR^n$ and $\BR^{n \times n}$ 
be set of real numbers, 
real $n$-dimensional column vectors and real $n \times n$ matrices, 
respectively. 
Also denote by 
$\BR_+$, $\BR^n_+$ and $\BR^{n \times n}_+$, 
the subsets of entrywise non-negative elements in 
$\BR$, $\BR^n$ and $\BR^{n \times n}$, 
respectively. 
We provide each space of
$\BR$, $\BR^n$ and $\BR^{n \times n}$ 
with the componentwise relation $\le$ and $<$ respectively. 
For any set $X$, we will denote a fuzzy set $\T{a}$ on $X$ 
by its membership function $\T{a} : X \to [0,1]$. 
Denote by $\C{F}(X)$ the set of all fuzzy sets on $X$. 
For the theory of fuzzy sets, refer to 
Zadeh\cite{Zad65} and Nov{\'a}k\cite{Nov89}. 
The $\af$-cut $(\af \in [0,1])$ of the fuzzy set $\T{a} \in \C{F}(X)$ 
is defined as 
$$
   \T{a}_{\af} := \{ x \in X \mid \T{a}(x) \ge \af \} \ (\af > 0)
   \quad \mbox{and} \quad
   \T{a}_0 := {\rm cl} \{ x \in X \mid \T{a}(x) > 0 \},
$$
where \lq\lq{\rm cl}\rq\rq \  denote the closure of the set.
For any interval $Y$ in $\BR$,
$\T{a} \in \C{F}(Y)$ is called a fuzzy number on $Y$ 
if $\T{a}$ has the following properties (i) $-$ (iv):
%
\vspace*{-0.5em}
\begin{description}
  \item[(i)]
   $\T{a}$ is normal, i.e., 
   there exists an $x_0 \in Y$ with $\T{a}(x_0)=1$;
%
%\vspace*{-0.5em}
  \item[(ii)]
   $\T{a}$ is convex, i.e., 
   $\T{a}(\af x + (1-\af)y) \ge \T{a}(x) \wedge \T{a}(y)$ 
   for all $x, y \in Y$ and $\af \in [0,1]$,
   where $a \wedge b = \min \{ a,b\}$;
%
%\vspace*{-0.5em}
  \item[(iii)]
   $\T{a}$ is upper semi-continuous;
%
%\vspace*{-0.5em}
  \item[(iv)]
   $\T{a}_0$ is a compact subset of $Y$.
\end{description}

%\vspace*{-0.5em}
Denote by $\C{F}_{c}(Y)$ the set of all fuzzy numbers on $Y$.
Let $\C{C}(Y)$ be the set of all closed and bounded intervals in $Y$.
We note that $\T{a} \in \C{F}_{c}(Y)$ means $\T{a}_{\af} \in \C{C}(Y)$ 
for all $\af \in [0,1]$.
Let $\C{F}_{c}(Y)^n$ be the set of all $n$-dimensional column vectors 
whose elements are in $\C{F}_{c}(Y)$, i.e., 
$$
  \C{F}_{c}(Y)^n := \{ \T{u} = (\T{u}_1, \T{u}_2, \ldots, \T{u}_n)' \mid 
  \T{u}_i \in \C{F}_{c}(Y) \ (1 \le i \le n) \},
$$
where $d'$ denotes the transpose of a vector $d$. 

Let $S := \{ 1,2,\ldots,n \}$ and $\C{P}(S)$ the set of all probability 
distributions on $S$, that is, 
$$
  \C{P}(S) := \{ p = (p_1, p_2, \ldots, p_n ) \mid 
  p_j \ge 0 \ (1 \le j \le n), \sum_{j=1}^{n}p_j = 1 \}.
$$
From any 
$\T{p} = (\T{p}_1, \T{p}_2, \ldots, \T{p}_n)' \in \C{F}_{c}([0,1])^n$, 
we will construct the fuzzy set 
$[\T{p}] = [\T{p}_1, \T{p}_2, \ldots, \T{p}_n]$ on $\C{P}(S)$ 
by the following membership function:
$$
  [\T{p}](p) = \min_{1 \le j \le n}\{ \T{p}_j(p_j) \}
  \quad \mbox{for any \ }
  p = (p_1, p_2, \ldots, p_n) \in \C{P}(S).
\eqno{(2.1)}
$$
The above definition will be extended to the case of stochastic matrices.
Let $\C{P}(S/S)$ be the set of all stochastic matrices on $S$, that is, 
$$
  \C{P}(S/S) := \{ Q = (q_{ij}) \mid 
  q_{ij} \ge 0, \sum_{j=1}^{n} q_{ij} = 1 \ (1 \le i \le n) \}.
$$
%\vspace*{-0.5em}
For any $\T{q}_{i} = (\T{q}_{i1}, \T{q}_{i2}, \ldots, \T{q}_{in}) \in 
\C{F}_c([0,1])^n \ (1 \le i \le n)$, 
we define the fuzzy set 
$\T{Q} = [\T{q}_1, \T{q}_2, \ldots, \T{q}_n]'$ on $\C{P}(S/S)$ 
by the following membership function: 
$$
  \T{Q}(Q) := \min_{1 \le i \le n} \{ [\T{q}_i](q_i) \},
\eqno{(2.2)}
$$
where 
$Q = (q_1, q_2, \ldots, q_n)' \in \C{P}(S/S)$, 
$q_i = (q_{i1}, q_{i2}, \ldots, q_{in}) \in \C{P}(S)$ and 
$[\T{q}_i]$ is the fuzzy set on $\C{P}(S)$ defined by $(2.1)$.

In order to describe the structural properties on the fuzzy sets 
defined in $(2.1)$ and $(2.2)$, 
we need the concept of intervals of matrices. 
For the detail, refer to \cite{Har98,KSHH98,Nen84}.
For any nonnegative vector 
$ \U{q} = (\U{q}_1, \U{q}_2, \ldots, \U{q}_n)$ and 
$ \O{q} = (\O{q}_1, \O{q}_2, \ldots, \O{q}_n) \in \BR^n_+$  
with $\U{q} \le \O{q}$, 
we define the set of probability distributions 
$\langle \U{q}, \O{q} \rangle \subset \C{P}(S)$ by 
$$
  \langle \U{q}, \O{q} \rangle := \{ 
  p = (p_1, p_2, \ldots, p_n) \in \C{P}(S) \mid \U{q} \le p \le \O{q} \}. 
\eqno{(2.3)}
$$
Similarly, for 
$\U{Q} = (\U{q}_{ij}), \O{Q} = (\O{q}_{ij}) \in \BR_+^{n \times n}$ 
with $\U{Q} \le \O{Q}$, 
we define the set of stochastic matrices 
$\langle \U{Q}, \O{Q} \rangle \subset \C{P}(S/S)$ by
$$
  \langle \U{Q}, \O{Q} \rangle := \{ 
  Q \in \C{P}(S/S) \mid \U{Q} \le Q \le \O{Q} \}.
\eqno{(2.4)}
$$

\noindent
{\bf Lemma 2.1} (\cite{Har98}). \ 
{\sl 
For any $\U{Q}, \O{Q} \in \BR_+^{n \times n}$ with $\U{Q} \le \O{Q}$ 
and $\langle \U{Q}, \O{Q} \rangle \not= \emptyset$, 
$\langle \U{Q}, \O{Q} \rangle$ is a polyhedral convex set 
in the vector space $\BR^{n \times n}$.
}


For any $\T{a} \in \C{F}_c([0,1])$, 
noting $\T{a}_{\af} \in \C{C}([0,1]) \ (0 \le \af \le 1)$,
it will be denoted by 
$\T{a}_{\af} = [ \min \T{a}_{\af}, \max \T{a}_{\af} ]$.
The structural property of the fuzzy sets defined in (2.1) and (2.2) 
is given, whose proof is done by using Lemma 2.1.


\noindent
{\bf Lemma 2.2}. \ 
{\sl 
For any $\T{q}_i \in \C{F}_c([0,1])^n \ (1 \le i \le n)$, 
let $\T{Q} = [\T{q}_1, \T{q}_2, \ldots, \T{q}_n]'$ 
be a fuzzy set on $\C{P}(S/S)$ defined by (2.1). 
Then, the $\af$-cut of $\T{Q} \ (0 \le \af \le 1)$ is a 
polyhedral convex subset of $\C{P}(S/S)$ and given by 
$$
  \T{Q}_\af = \langle \U{Q}_\af, \O{Q}_\af \rangle, \quad 
  \mbox{where \ } 
  \U{Q}_\af = \big( \min (\T{q}_{ij})_\af \big) \mbox{\ and \ }
  \O{Q}_\af = \big( \max (\T{q}_{ij})_\af \big).
\eqno{(2.5)}
$$
}
\Pf 
Since $\T{q}_{ij} \in \C{F}_c([0,1])$, 
the $\af$-cut $(\T{q}_{ij})_\af$  belongs to $\C{C}([0,1])$.
By (2.1) and (2.2), we observe that 
$$
  \T{Q}_\af = \{ Q = (q_{ij}) \in \C{P}(S/S) \mid 
            q_{ij} \in (\T{q}_{ij})_\af {\ }(1 \le i, j \le n) \},
$$
which implies that (2.5) holds.
Thus, by Lemma 2.1, $\T{Q}_\af$ has the required property. 
\hfill \qed


If 
$\BS{u} = ([a_1, b_1], [a_2, b_2], \ldots, [a_n, b_n])' \in \C{C}(\BR_+)^n$, 
$\BS{u}$ will be denoted by $\BS{u} = [a, b]$, 
where $a = (a_1, a_2, \ldots, a_n)'$, $b = (b_1, b_2, \ldots, b_n)'$ 
and $[a, b] = \{ x \in \BR_+^n \mid a \le x \le b \}$.
For any $\BS{u} \in \C{C}(\BR_+)^n$ and 
$\U{Q}, \O{Q} \in \BR_+^{n \times n}$ with $\U{Q} \le \O{Q}$ and 
$\langle \U{Q}, \O{Q} \rangle \not= \emptyset$, 
we define their product by 
$$
\langle \U{Q}, \O{Q} \rangle \BS{u} = 
  \{ Q{u} \mid Q \in \langle \U{Q}, \O{Q} \rangle , u \in \BS{u} \}.
\eqno{(2.6)}
$$ 

%\bigskip
\noindent
{\bf Lemma 2.3} (Lemma 1.4 in \cite{KSHH98}). \ 
{\sl 
%\vspace*{-0.5em}
$$
  \langle \U{Q}, \O{Q} \rangle \BS{u} \in \C{C}(\BR_+)^n 
  \mbox{\quad for all \ } \BS{u} \in \C{C}(\BR_+)^n.
$$
}
%
%\vspace*{-0.5em}
The following arithmetical notation is used in the sequel. 
Let 
$\T{Q} = [\T{q}_1, \T{q}_2, \ldots, \T{q}_n]'$ 
be a fuzzy set on $\C{P}(S/S)$ with 
$\T{q}_i \in \C{F}([0,1])^n \ (1 \le i \le n)$. Then, for 
$\T{\BS{u}} = (\T{u}_1, \T{u}_2, \ldots, \T{u}_n)' \in \C{F}_c(\BR_+)^n$, 
$\T{Q} \T{\BS{u}} \in \C{F}(\BR_+^n)$ 
is defined as follows:
$$
  (\T{Q}\T{\BS{u}})(x) = 
  \max_{\scriptsize x=Qu \atop Q \in \C{P}(S/S), u \in \BR_+^n}
  \{ \T{Q}(Q) \wedge \T{\BS{u}}(u) \}, 
  \mbox{\quad for \ } x \in \BR_+^n,
\eqno{(2.7)}
$$
where
$$
  \T{\BS{u}}(u) = \min_{1 \le i \le n}
  \{ \T{u}_i(u_i) \}
  \mbox{\quad with\quad } u = (u_1, u_2, \ldots, u_n) \in \BR_+^n.
\eqno{(2.8)}
$$
% Then, the following holds.

%\bigskip
\noindent
{\bf Lemma 2.4}. \ 
{\sl 
For any 
$\T{\BS{u}} = (\T{u}_1, \T{u}_2, \ldots, \T{u}_n)' \in \C{F}_c(\BR_+)^n$, 
we have:
%
%\vspace*{-0.5em}
\begin{description}
  \item[\rm (i)]
   $(\T{Q}\T{\BS{u}})_\af = \T{Q}_\af \T{\BS{u}}_\af$ \ for \ 
   $\af \in [0, 1]$;
\vspace*{0.75ex}
%\vspace*{-0.5em}
  \item[\rm (ii)]
   $\T{Q}\T{\BS{u}} \in \C{F}_c(\BR_+)^n$.
\end{description}
}
\Pf
By (2.7) we get 
$(\T{Q}\T{\BS{u}})_\af = 
\{ Qu \mid q \in \T{Q}_\af, u \in \T{\BS{u}}_\af \}$. 
From (2.8) it holds $\T{\BS{u}}_\af \in \C{C}(\BR_+)^n$, 
so that (i) follows by the definition (2.6).
Also, (ii) follows obviously from Lemma 2.2 and 2.3. \hfill \qed

%\bigskip
The addition and the scalar multiplication on $\C{F}_c(\BR_+)$ 
are defined as follows: 
For $\T{a}, \T{b} \in \C{F}_c(\BR_+)$ and $\ld \in \BR_+$, define 
$$
  (\T{a} + \T{b})(x) := 
  \sup_{\scriptsize x_1, x_2 \in \BR_+ \atop x_1+x_2=x}
  \{ \T{a}(x_1) \wedge \T{b}(x_2) \},
$$
$$
  \ld \T{a}(x) :=
     \left\{
       \begin{array}{ll}
        \T{a}(x/\ld) & \mbox{if} \ \ld>0 \\
        I_{\{0\}}(x) & \mbox{if} \ \ld=0
       \end{array}
       \quad (x \in \BR_+),
     \right.
$$
provided that $I_A$ is the indicator of a set $A$. 
It is easily shown that, for $\af \in [0,1]$, 
$$
  (\T{a} + \T{b})_\af = \T{a}_\af +\T{b}_\af 
  \mbox{\quad and \quad}
  (\ld\T{a})_\af = \ld\T{a}_\af,
$$
where the operation on sets is defined ordinary as 
$A + B := \{ x+y \mid x \in A, y \in B \}$ and 
$\ld A = \{ \ld x \mid x \in A \}$ for $A, B \subset \BR_+$.
The above operations are extended to those on $\C{F}_c(\BR_+)^n$ 
as follows: 
For $\T{\BS{u}} = (\T{u}_1, \T{u}_2, \ldots, \T{u}_n)'$,
$\T{\BS{v}} = (\T{v}_1, \T{v}_2, \ldots, \T{v}_n)' \in \C{F}_c(\BR_+)^n$, 
$$
  \T{\BS{u}} + \T{\BS{v}} = 
  (\T{u}_1 + \T{v}_1, \T{u}_2 + \T{v}_2, \ldots, \T{u}_n + \T{v}_n)'
  \mbox{\quad and \quad}
  \ld\T{\BS{u}} = (\ld\T{u}_1, \ld\T{u}_2, \ldots, \ld\T{u}_n)'.
$$
For $a = (a_1, a_2, \ldots, a_n)' \in \BR_+^n$, 
$I_{\{a\}} = (I_{\{a_1\}}, I_{\{a_2\}}, \ldots, I_{\{a_n\}}) 
\in \C{F}_c(\BR_+)^n$ and $I_{\{a\}} + \T{\BS{u}}$ 
is described simply by $a +\T{\BS{u}}$.
The Hausdorff metric on $\C{C}(\BR_+)$ is denoted by $\delta$, i.e., 
$$
  \delta ([a,b], [c,d]) := |a-c| \vee |b-d|
  \mbox{\quad for \ }
  [a,b], [c,d] \in \C{C}(\BR_+),
$$
where $x \vee y = \max \{ x, y \}$ for $x, y \in \BR$.
This metric can be extended to $\C{F}_c(\BR_+)^n$ by 
$$
  \delta (\T{\BS{u}}, \T{\BS{v}}) = 
  \max_{1 \le i \le n} \sup_{\af \in [0,1]}
  \delta ((\T{u}_{i})_{\af}, (\T{v}_{i})_{\af})
$$
for 
$\T{\BS{u}} =  (\T{{u}}_1, \T{{u}}_2, \ldots, \T{{u}}_n)'$,
$\T{\BS{v}} =  (\T{{v}}_1, \T{{v}}_2, \ldots, \T{{v}}_n)' 
\in \C{F}_c(\BR_+)^n$.
Then, it is known (c.f.\cite{Kra66}) that the metric space 
$(\C{F}_c(\BR_+)^n, \delta)$ is complete.


%%%%%%%%%%%%%%%%%%%%%%%%
%      Section 3       %
%%%%%%%%%%%%%%%%%%%%%%%%
\section{The fuzzy description of MDPs} % ------------------
In order to deal with the vague data and flexible requirements 
for nonhomogenuous MDPs we shall use a fuzzy set representation.
%
Let $S$ and $A$ be finite sets denoted by 
$S = \{ 1, 2, \ldots, n \}$ and $A = \{ 1, 2, \ldots, k \}$.
Our sequential decision model consists of four objects: 
$$
  ( S, A, \{ \T{q}_{ij}(a) \in \C{F}_c([0,1]), 
  \ i, j \in S, \ a \in A \}, r ),
$$
where $r = r(i,a)$ is a function on $S \times A$ with $r \ge 0$. 
We interpret $S$ as the set of states of some system 
and $A$ as the set of actions available at each state. 
We denote by $F$ the set of all functions from $S$ to $A$. 
%
For any $f \in F$, 
we define the fuzzy set $\T{Q}(f)$ on $\C{P}(S/S)$ as follows: 
$$
  \T{Q}(f) := [ \T{q}_1(f), \T{q}_2(f), \ldots, \T{q}_n(f) ]'
  \quad \mbox{where}
\eqno{(3.1)}
$$
$$
  \T{q}_i(f) := 
  [ \T{q}_{i1}(f(i)), \T{q}_{i2}(f(i)), \ldots, \T{q}_{in}(f(i)) ]
  \quad ( 1 \le i \le n ).
\eqno{(3.2)}
$$
Note that the basic notations of (3.1) and (3.2) are 
defined in (2.1) and (2.2).

A policy $\pi$ is a sequence $(f_1, f_2, \ldots )$ of functions with 
$f_t \in F \ (t \ge 1)$.
Let $\Pi$ be the class of policies. 
We denote by $f^{\infty}$ the policy $(h_1, h_2, \ldots )$ with 
$h_t = f$ for all $t \ge 1$ and some $f \in F$. 
Such a policy is called stationary and denoted simply by $f \in F$. 
The set of all stationary policies will be denoted by $\Pi_F$.
%
For any $f \in F$, let $r(f)$ be an $n$-dimensional column vector 
whose $i$-th element is $r(i,f(i))$. 
Applying Zadeh's extension principle(cf. \cite{Nov89}), 
the fuzzy expected total discounted reward up to time $T$ 
from a policy $\pi$ is a element of $\C{F}(\BR_+)^n$ 
and defined as follows: 
$$
  \T{\psi}_{T}(\pi) := 
  ( \T{\psi}_{T}(1,\pi), \T{\psi}_{T}(2,\pi), \ldots, 
  \T{\psi}_{T}(n,\pi) )'
\eqno{(3.3)}
$$
and
$$
  \T{\psi}_{T}(i,\pi)(x) 
  := \max \{ \min_{1 \le t \le T} \T{Q}(f_t)(Q_t) \}
\eqno{(3.4)}
$$
for all  $x \in \BR_+,  1 \le i \le n$, where the maximum 
is taken over 
\begin{align}
  \{ Q_1, Q_2, \ldots, Q_T  \mid \ 
  & x = ( r(f_1) + \beta Q_1 r(f_2) + \cdots \notag \\ 
  &  + \beta^T Q_1 Q_2 \cdots Q_T r(f_{T+1}))_{i}, \tag{3.5} \\
  & Q_t \in \C{P}(S/S) \ (1 \le t \le T) \}          \notag
\end{align}            
and $\beta$ is a discounted factor with $0 < \beta < 1$. 

%\bigskip
\noindent
{\bf Lemma 3.1} \ 
{\sl
For any policy $\pi \in \Pi$, we have: 
%
%\vspace*{-0.5em}
\begin{description}
  \item[\rm (i)]
   $\T{\psi}_{T}(\pi) \in \C{F}_c(\BR_+)^n $ \ 
   for all $T \ge 1$;
\vspace*{0.5ex}
  \item[\rm (ii)]
   $\{ \T{\psi}_{T}(\pi) \}$ is a Caushy sequence.
\end{description}
}
\Pf
We show that, for example, (i) holds for $T=2$. 
By (3.3) -- (3.5),
\begin{align}
  &( \T{\psi}_{T}(1,\pi)_{\af}, 
     \T{\psi}_{T}(2,\pi)_{\af}, 
     \ldots, 
     \T{\psi}_{T}(n,\pi)_{\af} )'                                 \notag   \\ 
  &= \{ r(f_1) + \beta Q_1 r(f_2) + \beta^2 Q_1 Q_2 r(f_3) \mid 
  Q_i \in \T{Q}(f_i)_\af, \ 1 \le i \le 2 \}                      \notag   \\ 
  &= r(f_1)+\beta \T{Q}(f_1)_\af (r(f_2) +\beta \T{Q}(f_2)_\af r(f_2)). \notag 
\end{align}
Therefore, Lemma 2.2 and 2.3 it follows that
$$
  ( \T{\psi}_{T}(1,\pi)_\af, 
    \T{\psi}_{T}(2,\pi)_\af, 
    \ldots, 
    \T{\psi}_{T}(n,\pi)_\af )' \in \C{C}(\BR_+)^n,
$$
which implies (i) for $T=2$. 
By the same method as the case of $T=2$, 
we can prove (i) for any $T$. 
Also, (ii) follows easily from the properties of the Hausdorff metric 
and the existence of the discount factor $\beta \ (0 < \beta < 1)$.
\hfill \qed

%\bigskip
By Lemma 3.1, we can define the infinite horizon 
fuzzy expected discounted reward(FEDR) from a policy $\pi$ by
$$
  \T{\psi}(\pi) := 
  \lim_{T \to \infty} \T{\psi}_T(\pi).
$$
Here, we will give a partial order $\inc$ on $\C{C}(\BR_+)$ 
by the definition: 
For $[a,b]$, $[c,d] \in \C{C}(\BR_+)$, 
\begin{align*}
  [a,b] \ \inc \ [c,d]  
  & \mbox{\quad if \ } a \le c \mbox{\ and \ } b \le d,   \\
  [a,b] \prec [c,d]     
  & \mbox{\quad if \ }
  [a,b] \ \inc \ [c,d] \mbox{\ and \ } [a,b] \not= [c,d].
\end{align*}
This partial order $\inc$ on $\C{C}(\BR_+)$ 
is extended to that of $\C{F}_c(\BR_+)$, 
called a fuzzy max order, as follows:
For $\T{u}$, $\T{v} \in \C{F}_c(\BR_+)$,
\begin{align*}
  \T{u} \ \inc \ \T{v} 
  & \mbox{\quad if \ }
  \T{u}_\af \ \inc \ \T{v}_\af \mbox{\ for all \ } \af \in [0,1], \\
  \T{u} \prec \T{v} 
  & \mbox{\quad if \ }
  \T{u} \ \inc \ \T{v} \mbox{\ and \ } \T{u} \not= \T{v}.
\end{align*}
Also, as a further extension, the partial order on $\C{F}_c(\BR_+)^n$ 
is given by the definition: For 
$\T{\BS{u}} = (\T{u}_1, \T{u}_2, \ldots, \T{u}_n)'$, 
$\T{\BS{v}} = (\T{v}_1, \T{v}_2, \ldots, \T{v}_n)' \in \C{F}_c(\BR_+)^n$,
\begin{align*}
  \T{\BS{u}} \ \inc \ \T{\BS{v}} 
  & \mbox{\quad if \ }
  \T{u}_i \ \inc \ \T{v}_i \mbox{\ for all \ } i = 1, 2, \ldots, n, \\
  \T{\BS{u}} \prec \T{\BS{v}} 
  & \mbox{\quad if \ } \T{\BS{u}} \ \inc \ \T{\BS{v}} 
  \mbox{\ and \ } \T{\BS{u}} \not= \T{\BS{v}}.
\end{align*}
Our problem is to maximize the $\T{\psi}(\pi)$ over all $\pi \in \Pi$ 
with respect to the partial order $\inc$.

The following lemma is used in the sequel whose proof is easily done.

%\bigskip
\noindent
{\bf Lemma 3.2} \ 
{\sl 
Let a sequence $\{ \T{\BS{u}}_n \} \subset \C{F}_c(\BR_+)^n$ 
be such that $\T{\BS{u}}_1\ \inc \  \T{\BS{u}}_2 \ \inc \  \cdots$, and 
$\lim_{k \to \infty} \T{\BS{u}}_k = \T{\BS{u}}$ for some 
$\T{\BS{u}} \in \C{F}_c(\BR_+)^n$. 
Then, it holds that $\T{\BS{u}}_1 \ \inc \ \T{\BS{u}}$.
}


%%%%%%%%%%%%%%%%%%%%%%%%
%      Section 4       %
%%%%%%%%%%%%%%%%%%%%%%%%
\section{Stationary policies and operators} % -------------------------
In this section, the infinite horizon FEDR from a stationary policy 
is given as a unique fixed point of a corresponding operator. 
Associated with each function $f \in F$ is a corresponding operator 
$U(f) : \C{F}_c(\BR_+)^n \to \C{F}_c(\BR_+)^n$ 
defined as follows: For $\T{\BS{u}} \in \C{F}_c(\BR_+)^n$, 
$$
  U_f \T{\BS{u}} = r(f) + \beta \T{Q}(f)\T{\BS{u}},
\eqno{(4.1)}
$$
where the arithmetics in (4.1) are defined in (2.7). 
Note that from Lemma 1.4 $U_f$ is well-defined.

For any policy $\pi = (f_1, f_2, \ldots )$, 
let $\pi^{-l} = (f_{l+1}, f_{l+2}, \ldots)$ for each $l \ge 1$.
The sequence $\{ \T{\psi}_T(\pi) \}_{T=1}^{\infty}$ 
is recursively described.

%\bigskip
\noindent
{\bf Lemma 4.1}\ 
{\sl
For any policy $\pi = (f_1, f_2, \ldots)$, we have 
$$
  \T{\psi}_T(\pi) = U_{f_1}U_{f_2}\cdots U_{f_l}\T{\psi}_{T-l}(\pi^{-l})
  \mbox{\ for each \ } l \ge 1.
\eqno{(4.2)}
$$
}
\Pf
From (3.3)--(3.5) and Lemma 1.4 (i), we get 
$\T{\psi}_2(i,\pi)_\af = 
  ( r(f_1) + \beta \T{Q}(f_1)r(f_2))_\af 
  = r(f_1) + \beta \T{Q}(f_1)_{\af}r(f_2)$
for each $\af \in [0,1]$.
Since $\T{\psi}_1(\pi^{-1}) = r(f_2)$, (4.2) holds for $T=2$ and $l=1$. 
By induction on $T$ and $l$, we can easily proved (4.2).
\hfill \qed

%\bigskip
\noindent
{\bf Lemma 4.2.}\ 
{\sl
Let $f \in F$. Then we have: 
%
%\vspace*{-0.5em}
\begin{description}
  \item[\rm (i)]
   $U_f$ is a contraction with modulus $\beta$, i.e., 
   for $\T{\BS{u}}, \T{\BS{v}} \in \C{F}_c(\BR_+)^n$, 
   $$\delta (U_f \T{\BS{u}}, U_f \T{\BS{v}}) \le 
     \beta \delta(\T{\BS{u}}, \T{\BS{v}}),$$  
%
%\vspace*{-0.5em}
  \item[\rm (ii)]
   $U_f$ is monotone, i.e., $\T{\BS{u}} \ \inc \  \T{\BS{v}}$ implies 
   $U_f \T{\BS{u}} \ \inc \  U_f \T{\BS{v}}$.
\end{description}
}
\Pf
For any $\T{\BS{u}}, \T{\BS{v}} \in \C{F}_c(\BR_+)^n$, 
from the property of the Hausdorff metric, it holds 
$
\delta (U_f \T{\BS{u}}, U_f \T{\BS{v}}) \le 
\beta \delta(\T{Q}(f)\T{\BS{u}}, \T{Q}(f)\T{\BS{v}}).
$
Using Lemma 2.4 (i), we get
$$
  \delta((\T{Q}(f)\T{\BS{u}})_\af, (\T{Q}(f)\T{\BS{v}})_\af) = 
  \delta(\T{Q}(f)_\af \T{\BS{u}}_\af, \T{Q}(f)_\af\T{\BS{v}}_\af) \le 
  \delta(\T{\BS{u}}_\af, \T{\BS{v}}_\af) .
$$
So, we have 
$
\delta (U_f \T{\BS{u}}, U_f \T{\BS{v}}) \le 
   \beta \delta(\T{\BS{u}}, \T{\BS{v}}),
$
which implies (i). 
Also, (ii) follows obviously. 
\hfill \qed


%\bigskip
By Lemma 3.1, 
$\T{\psi}_T(f) = U_f\T{\psi}_{T-1}(f)$ for all $T \ge 2$. 
As $T \to \infty$ in the above, 
$\T{\psi}(f)$ is a fixed point of $U_f$. 
Thus, the following characterization of $\T{\psi}(f)$ are immediate
and so its proof is omitted.

\noindent
{\bf Theorem 4.1.} \ 
{\sl 
For any $f \in F$, $\T{\psi}(f)$ is a unique solution of 
the following fuzzy inclusion:
$$
  \T{\BS{u}} = U_f \T{\BS{u}}, \quad 
  \T{\BS{u}} \in \C{F}_c(\BR_+)^n.
\eqno{(4.3)}
$$
}
%
%\vspace*{-0.5em}

Applying Lemma 2.4 (i), (4.3) can be rewritten by the following 
$\af$-cut interval equation:
$$
  \T{\BS{u}}_\af = r(f) + \beta \T{Q}(f)_\af \T{\BS{u}}_\af, \ 
  0 \le \af \le 1,
\eqno{(4.4)}
$$
where $\T{\BS{u}}_\af = 
\big( (
\T{u}_1)_\af, (\T{u}_2)_\af, \ldots, (\T{u}_n)_\af 
\big)' \in \C{C}(\BR_+)^n$ 
and 
$\T{Q}(f)_\af = \langle \U{Q}_\af, \O{Q}_\af \rangle$ 
with $ \U{Q}_\af \le  \O{Q}_\af $.
By a contraction of $U_f$, the following holds. 

%\bigskip
\noindent
{\bf Corollary 4.1.} \ 
{\sl 
For any $f \in F$ and $\T{\BS{u}} \in \C{F}_c(\BR_+)^n$, \ \ 
$\displaystyle{
\T{\psi}(f) = \lim_{l \to \infty}U_{f}^{l}\T{\BS{u}}.
}$
}


As a simple example, we consider a fuzzy treatment for 
a machine maintenance problem(\cite{MinOsa70}, p.1, p.17--18). 

\noindent
{\bf Example 1.} %(A machine maintenance problem) 
A machine can be operated synchronously, say, once an hour. 
At each period there are two states; 
one is operating(state 1), and the other is in failure(state 2).
If the machine fails, it can be restored to perfect functioning by repair. 
At each period, if the machine is running, 
we earn the return of \$ 3.00 per period; 
the fuzzy set of probability of being in state 1 at the next step is 
(0.6/0.7/0.8) 
and that of the probability of moving to state 2 is 
(0.2/0.3/0.4), 
where for any $0 \le a < b < c \le 1$, 
the fuzzy number $(a/b/c)$ on $[0,1]$ is defined by
$$
(a/b/c)(x) = 
  \left\{
  \begin{array}{ll}
   (x-a)/(b-a) \vee 0 & \mbox{if \ } 0 \le x \le b, \\
   (x-c)/(b-c) \vee 0 & \mbox{if \ } b \le x \le 1.
  \end{array}
  \right.
$$
If the machine is in failure, 
we have two actions to repair the failed machine; 
one is a usual repair, denoted by 1, 
that yields the cost of \$ 1.00(that is, a return of $-$\$1.00) 
with the fuzzy set 
(0.3/0.4/0.5) 
of the probability moving in state 1 
and the fuzzy set 
(0.5/0.6/0.7) 
of the probability being in state 2;  
another is a rapid repair, denoted by 2, 
that requires the cost of \$2.00(that is, a return of $-$\$2.00) 
with the fuzzy set 
(0.5/0.6/0.7) 
of the probability moving in state 1 
and the fuzzy set 
(0.3/0.4/0.5) 
of the probability being in state 2. 

For the model considered, $S = \{ 1, 2 \}$ 
and there exists two stationary policies, $F = \{ f_1, f_2 \}$ 
with $f_1(2) = 1$ and $f_2(2) =2$, 
where $f_1$ denotes a policy of the usual repair and 
$f_2$ a policy of the rapid repair.
The state transition diagrams of two policies are shown in Figure 1.


\begin{center}
% Fig.1 --------------------------------------------------------------
%\input fa.txt %------------------------------------------------------
\vspace*{-2.5cm} 

\begin{figure}[htbp]
\unitlength 0.001in
\hspace*{1.85cm}
\begin{picture}(50,100)
% CIRCLE 2 0 3 0
% 4 100 180 114 182 115 180 115 180
%
\special{pn 8}%
\special{ar 1000 1400 141 141  0.0000000 6.2831853}%
% CIRCLE 2 0 3 0
% 4 260 180 274 182 275 180 275 180
%
\special{pn 8}%
\special{ar 2600 1400 141 141  0.0000000 6.2831853}%
% ELLIPSE 2 0 3 0
% 4 180 181 102 204 109 191 250 190
%
\special{pn 8}%
\special{ar 1800 1410 780 230  0.4168981 2.6949888}%
% SPLINE2 2 0 3 0
% 42 1087 1697 1106 1685 1127 1674 1151 1662 1177 1652 1204 1642 1234 1632	1265 1623 1299 1614 1333 1606 1370 1598 1407 1591 1446 1585 1487 1579 1528	1574 1570 1570 1613 1567 1656 1564 1700 1562 1744 1561 1788 1560 1833 1560	1877 1561 1921 1563 1965 1565 2008 1568 2050 1572 2092 1577 2133 1582 2172	1588 2211 1594 2248 1602 2283 1610 2318 1618 2350 1627 2381 1636 2409 1646	2436 1657 2461 1668 2483 1679 2503 1691 2503 1691
%
\special{pn 8}%
\special{pa 1087 1297}%
\special{pa 1114 1280}%
\special{pa 1143 1266}%
\special{pa 1172 1254}%
\special{pa 1202 1243}%
\special{pa 1233 1232}%
\special{pa 1263 1223}%
\special{pa 1294 1215}%
\special{pa 1326 1208}%
\special{pa 1357 1201}%
\special{pa 1388 1194}%
\special{pa 1420 1189}%
\special{pa 1451 1184}%
\special{pa 1483 1180}%
\special{pa 1515 1175}%
\special{pa 1547 1172}%
\special{pa 1578 1169}%
\special{pa 1610 1167}%
\special{pa 1642 1165}%
\special{pa 1674 1163}%
\special{pa 1706 1162}%
\special{pa 1738 1161}%
\special{pa 1770 1160}%
\special{pa 1802 1160}%
\special{pa 1834 1160}%
\special{pa 1866 1161}%
\special{pa 1898 1162}%
\special{pa 1930 1163}%
\special{pa 1962 1165}%
\special{pa 1994 1167}%
\special{pa 2026 1170}%
\special{pa 2058 1173}%
\special{pa 2089 1177}%
\special{pa 2121 1180}%
\special{pa 2153 1185}%
\special{pa 2185 1190}%
\special{pa 2216 1195}%
\special{pa 2247 1202}%
\special{pa 2279 1209}%
\special{pa 2310 1216}%
\special{pa 2341 1224}%
\special{pa 2371 1233}%
\special{pa 2402 1243}%
\special{pa 2432 1255}%
\special{pa 2461 1268}%
\special{pa 2489 1283}%
\special{pa 2503 1291}%
\special{sp}%
% VECTOR 2 0 3 0
% 2 246 167 251 169
%
\special{pn 8}%
\special{pa 2460 1270}%
\special{pa 2510 1290}%
\special{fp}%
\special{sh 1}%
\special{pa 2510 1290}%
\special{pa 2456 1247}%
\special{pa 2460 1270}%
\special{pa 2441 1284}%
\special{pa 2510 1290}%
\special{fp}%
% VECTOR 2 0 3 0
% 2 114 193 109 191
%
\special{pn 8}%
\special{pa 1140 1530}%
\special{pa 1090 1510}%
\special{fp}%
\special{sh 1}%
\special{pa 1090 1510}%
\special{pa 1144 1553}%
\special{pa 1140 1530}%
\special{pa 1159 1516}%
\special{pa 1090 1510}%
\special{fp}%
% ELLIPSE 2 0 3 0
% 4 298 182 259 213 264 195 262 166
%
\special{pn 8}%
\special{ar 2980 1420 390 310  3.6486912 6.2831853}%
\special{ar 2980 1420 390 310  0.0000000 2.7017501}%
% SPLINE2 2 0 3 0
% 42 945 1941 916 1974 882 2006 843 2032 799 2055 753 2073 703 2086 653		2093 601 2095 549 2092 498 2083 449 2068 402 2049 361 2025 322 1997 289		1964 262 1930 240 1891 226 1852 218 1811 216 1770 222 1728 234 1688 254		1651 279 1614 311 1581 347 1552 388 1527 433 1507 481 1491 531 1480 583		1476 635 1477 686 1482 737 1494 785 1511 828 1532 869 1558 906 1588 936		1622 961 1658 961 1658
%
\special{pn 8}%
\special{pa 945 1541}%
\special{pa 924 1565}%
\special{pa 902 1588}%
\special{pa 878 1609}%
\special{pa 851 1627}%
\special{pa 823 1643}%
\special{pa 795 1657}%
\special{pa 765 1669}%
\special{pa 734 1679}%
\special{pa 703 1686}%
\special{pa 672 1691}%
\special{pa 640 1694}%
\special{pa 608 1695}%
\special{pa 576 1694}%
\special{pa 544 1691}%
\special{pa 512 1686}%
\special{pa 481 1678}%
\special{pa 451 1669}%
\special{pa 421 1657}%
\special{pa 392 1644}%
\special{pa 364 1627}%
\special{pa 338 1609}%
\special{pa 313 1589}%
\special{pa 291 1566}%
\special{pa 270 1541}%
\special{pa 252 1515}%
\special{pa 238 1486}%
\special{pa 227 1456}%
\special{pa 220 1425}%
\special{pa 216 1393}%
\special{pa 217 1361}%
\special{pa 222 1330}%
\special{pa 230 1299}%
\special{pa 243 1270}%
\special{pa 260 1242}%
\special{pa 278 1216}%
\special{pa 299 1192}%
\special{pa 323 1171}%
\special{pa 348 1151}%
\special{pa 375 1134}%
\special{pa 404 1119}%
\special{pa 433 1107}%
\special{pa 464 1096}%
\special{pa 494 1087}%
\special{pa 526 1081}%
\special{pa 558 1077}%
\special{pa 590 1076}%
\special{pa 622 1076}%
\special{pa 653 1078}%
\special{pa 685 1082}%
\special{pa 717 1088}%
\special{pa 747 1097}%
\special{pa 778 1108}%
\special{pa 807 1121}%
\special{pa 835 1136}%
\special{pa 862 1153}%
\special{pa 888 1172}%
\special{pa 912 1194}%
\special{pa 933 1218}%
\special{pa 951 1244}%
\special{pa 961 1258}%
\special{sp}%
% VECTOR 2 0 3 0
% 2 93 162 98 166
%
\special{pn 8}%
\special{pa 930 1220}%
\special{pa 980 1260}%
\special{fp}%
\special{sh 1}%
\special{pa 980 1260}%
\special{pa 940 1203}%
\special{pa 938 1227}%
\special{pa 915 1234}%
\special{pa 980 1260}%
\special{fp}%
% STR 2 0 3 0
% 2 35 127 35 144
% $(0.6,0.7,0.8)$
\put(360,-990){$(0.6/0.7/0.8)$}%
% STR 2 0 3 0
% 2 146 136 146 153
% $(0.2,0.3,0.4)$
\put(1500,-1060){$(0.2/0.3/0.4)$}%
% STR 2 0 3 0
% 2 337 172 337 189
% $(0.5,0.6,0.7)$
\put(3400,-1460){$(0.5/0.6/0.7)$}%
% STR 2 0 3 0
% 2 145 206 145 223
% $(0.3,0.4,0.5)$
\put(1500,-1800){$(0.3/0.4/0.5)$}%
% STR 2 0 3 0
% 2 92 172 92 189
% $1$
\put(950,-1440){$1$}%
% STR 2 0 3 0
% 2 250 171 250 188
% $2$
\put(2540,-1440){$2$}%
% VECTOR 2 0 3 0
% 2 265 198 262 194
%
\special{pn 8}%
\special{pa 2650 1580}%
\special{pa 2620 1540}%
\special{fp}%
\special{sh 1}%
\special{pa 2620 1540}%
\special{pa 2644 1605}%
\special{pa 2652 1583}%
\special{pa 2676 1581}%
\special{pa 2620 1540}%
\special{fp}%
\end{picture}

\vspace*{4cm}

\end{figure}

\vskip0.4cm %\hskip4.5cm  
{\bf (a)} Usual repair $f_1$
\end{center}


\begin{center}
%\input fb.txt %-------------------------------------------------------------
\begin{figure}[htbp]
\unitlength 0.001in
\vspace*{-2.0cm}
\hspace*{1.85cm}
\begin{picture}(50,100)
% CIRCLE 2 0 3 0
% 4 100 180 114 182 115 180 115 180
%
\special{pn 8}%
\special{ar 1000 1400 141 141  0.0000000 6.2831853}%
% CIRCLE 2 0 3 0
% 4 260 180 274 182 275 180 275 180
%
\special{pn 8}%
\special{ar 2600 1400 141 141  0.0000000 6.2831853}%
% ELLIPSE 2 0 3 0
% 4 180 181 102 204 109 191 250 190
%
\special{pn 8}%
\special{ar 1800 1410 780 230  0.4168981 2.6949888}%
% SPLINE2 2 0 3 0
% 42 1087 1697 1106 1685 1127 1674 1151 1662 1177 1652 1204 1642 1234 1632	1265 1623 1299 1614 1333 1606 1370 1598 1407 1591 1446 1585 1487 1579 1528	1574 1570 1570 1613 1567 1656 1564 1700 1562 1744 1561 1788 1560 1833 1560	1877 1561 1921 1563 1965 1565 2008 1568 2050 1572 2092 1577 2133 1582 2172	1588 2211 1594 2248 1602 2283 1610 2318 1618 2350 1627 2381 1636 2409 1646	2436 1657 2461 1668 2483 1679 2503 1691 2503 1691
%
\special{pn 8}%
\special{pa 1087 1297}%
\special{pa 1114 1280}%
\special{pa 1143 1266}%
\special{pa 1172 1254}%
\special{pa 1202 1243}%
\special{pa 1233 1232}%
\special{pa 1263 1223}%
\special{pa 1294 1215}%
\special{pa 1326 1208}%
\special{pa 1357 1201}%
\special{pa 1388 1194}%
\special{pa 1420 1189}%
\special{pa 1451 1184}%
\special{pa 1483 1180}%
\special{pa 1515 1175}%
\special{pa 1547 1172}%
\special{pa 1578 1169}%
\special{pa 1610 1167}%
\special{pa 1642 1165}%
\special{pa 1674 1163}%
\special{pa 1706 1162}%
\special{pa 1738 1161}%
\special{pa 1770 1160}%
\special{pa 1802 1160}%
\special{pa 1834 1160}%
\special{pa 1866 1161}%
\special{pa 1898 1162}%
\special{pa 1930 1163}%
\special{pa 1962 1165}%
\special{pa 1994 1167}%
\special{pa 2026 1170}%
\special{pa 2058 1173}%
\special{pa 2089 1177}%
\special{pa 2121 1180}%
\special{pa 2153 1185}%
\special{pa 2185 1190}%
\special{pa 2216 1195}%
\special{pa 2247 1202}%
\special{pa 2279 1209}%
\special{pa 2310 1216}%
\special{pa 2341 1224}%
\special{pa 2371 1233}%
\special{pa 2402 1243}%
\special{pa 2432 1255}%
\special{pa 2461 1268}%
\special{pa 2489 1283}%
\special{pa 2503 1291}%
\special{sp}%
% VECTOR 2 0 3 0
% 2 246 167 251 169
%
\special{pn 8}%
\special{pa 2460 1270}%
\special{pa 2510 1290}%
\special{fp}%
\special{sh 1}%
\special{pa 2510 1290}%
\special{pa 2456 1247}%
\special{pa 2460 1270}%
\special{pa 2441 1284}%
\special{pa 2510 1290}%
\special{fp}%
% VECTOR 2 0 3 0
% 2 114 193 109 191
%
\special{pn 8}%
\special{pa 1140 1530}%
\special{pa 1090 1510}%
\special{fp}%
\special{sh 1}%
\special{pa 1090 1510}%
\special{pa 1144 1553}%
\special{pa 1140 1530}%
\special{pa 1159 1516}%
\special{pa 1090 1510}%
\special{fp}%
% ELLIPSE 2 0 3 0
% 4 298 182 259 213 264 195 262 166
%
\special{pn 8}%
\special{ar 2980 1420 390 310  3.6486912 6.2831853}%
\special{ar 2980 1420 390 310  0.0000000 2.7017501}%
% SPLINE2 2 0 3 0
% 42 945 1941 916 1974 882 2006 843 2032 799 2055 753 2073 703 2086 653		2093 601 2095 549 2092 498 2083 449 2068 402 2049 361 2025 322 1997 289		1964 262 1930 240 1891 226 1852 218 1811 216 1770 222 1728 234 1688 254		1651 279 1614 311 1581 347 1552 388 1527 433 1507 481 1491 531 1480 583		1476 635 1477 686 1482 737 1494 785 1511 828 1532 869 1558 906 1588 936		1622 961 1658 961 1658
%
\special{pn 8}%
\special{pa 945 1541}%
\special{pa 924 1565}%
\special{pa 902 1588}%
\special{pa 878 1609}%
\special{pa 851 1627}%
\special{pa 823 1643}%
\special{pa 795 1657}%
\special{pa 765 1669}%
\special{pa 734 1679}%
\special{pa 703 1686}%
\special{pa 672 1691}%
\special{pa 640 1694}%
\special{pa 608 1695}%
\special{pa 576 1694}%
\special{pa 544 1691}%
\special{pa 512 1686}%
\special{pa 481 1678}%
\special{pa 451 1669}%
\special{pa 421 1657}%
\special{pa 392 1644}%
\special{pa 364 1627}%
\special{pa 338 1609}%
\special{pa 313 1589}%
\special{pa 291 1566}%
\special{pa 270 1541}%
\special{pa 252 1515}%
\special{pa 238 1486}%
\special{pa 227 1456}%
\special{pa 220 1425}%
\special{pa 216 1393}%
\special{pa 217 1361}%
\special{pa 222 1330}%
\special{pa 230 1299}%
\special{pa 243 1270}%
\special{pa 260 1242}%
\special{pa 278 1216}%
\special{pa 299 1192}%
\special{pa 323 1171}%
\special{pa 348 1151}%
\special{pa 375 1134}%
\special{pa 404 1119}%
\special{pa 433 1107}%
\special{pa 464 1096}%
\special{pa 494 1087}%
\special{pa 526 1081}%
\special{pa 558 1077}%
\special{pa 590 1076}%
\special{pa 622 1076}%
\special{pa 653 1078}%
\special{pa 685 1082}%
\special{pa 717 1088}%
\special{pa 747 1097}%
\special{pa 778 1108}%
\special{pa 807 1121}%
\special{pa 835 1136}%
\special{pa 862 1153}%
\special{pa 888 1172}%
\special{pa 912 1194}%
\special{pa 933 1218}%
\special{pa 951 1244}%
\special{pa 961 1258}%
\special{sp}%
% VECTOR 2 0 3 0
% 2 93 162 98 166
%
\special{pn 8}%
\special{pa 930 1220}%
\special{pa 980 1260}%
\special{fp}%
\special{sh 1}%
\special{pa 980 1260}%
\special{pa 940 1203}%
\special{pa 938 1227}%
\special{pa 915 1234}%
\special{pa 980 1260}%
\special{fp}%
% STR 2 0 3 0
% 2 35 127 35 144
% $(0.6,0.7,0.8)$
\put(360,-990){$(0.6/0.7/0.8)$}%
% STR 2 0 3 0
% 2 146 136 146 153
% $(0.2,0.3,0.4)$
\put(1500,-1060){$(0.2/0.3/0.4)$}%
% STR 2 0 3 0
% 2 337 172 337 189
% $(0.3,0.4,0.5)$
\put(3400,-1460){$(0.3/0.4/0.5)$}%
% STR 2 0 3 0
% 2 145 206 145 223
% $(0.5,0.6,0.7)$
\put(1500,-1800){$(0.5/0.6/0.7)$}%
% STR 2 0 3 0
% 2 92 172 92 189
% $1$
\put(950,-1440){$1$}%
% STR 2 0 3 0
% 2 250 171 250 188
% $2$
\put(2540,-1440){$2$}%
% VECTOR 2 0 3 0
% 2 265 198 262 194
%
\special{pn 8}%
\special{pa 2650 1580}%
\special{pa 2620 1540}%
\special{fp}%
\special{sh 1}%
\special{pa 2620 1540}%
\special{pa 2644 1605}%
\special{pa 2652 1583}%
\special{pa 2676 1581}%
\special{pa 2620 1540}%
\special{fp}%
\end{picture}

\vspace*{4cm}

\end{figure}

\vskip0.4cm %\hskip4.5cm 
{\bf (b)} Rapid repair $f_2$
%\vskip0.5cm %\hskip4.0cm 

{\bf Figure.1} \;Transition diagrams.
\end{center}

% Fig.1 ---------------------------------------------------------------------

%\bigskip
We easily observe that
$$
r(f_1) = 
         \left(
         \begin{array}{@{\,}c@{\,}}
         3      \\
         -1
         \end{array}
         \right)
\mbox{\ and \ }
\T{Q}(f_1) = 
         \left(
         \begin{array}{@{\,}cc@{\,}}
         (0.6/ 0.7/ 0.8)  &  (0.2/ 0.3/ 0.4)   \\
         (0.3/ 0.4/ 0.5)  &  (0.5/ 0.6/ 0.7)
         \end{array}
         \right),
$$
Now, applying Theorem 4.1, we can obtain the infinite horizon FEDR 
as a unique solution of (4.4). 
We observe that 
$$
\T{Q}(f_1)_\af = \Big\langle 
         \left(
         \begin{array}{@{\,}cc@{\,}}
           0.6 + 0.1\af  &  0.2 + 0.1\af   \\
           0.3 + 0.1\af  &  0.5 + 0.1\af
         \end{array}
         \right),
         \left(
         \begin{array}{@{\,}cc@{\,}}
           0.8 - 0.1\af  &  0.4 - 0.1\af   \\
           0.5 - 0.1\af  &  0.7 - 0.1\af
         \end{array}
         \right)
                 \Big\rangle.
$$
So, putting 
$\T{\psi}(f_1)_\af =[(x_1^{\af}, y_1^{\af})', (x_2^{\af}, y_2^{\af})']$, 
the $\af$-cut interval equations (4.4) with $\beta = 0.9$ become: 
$$
%\begin{array}{@{\,}l@{\,}}
\begin{array}{c}
  x_1^{\af} = 3 + 0.9 \{
        (0.6x_1^{\af} + 0.4x_2^{\af} + 0.1\af( x_1^{\af} - x_2^{\af})) \\
          \hspace{9em} \wedge
          (0.8x_1^{\af} + 0.2x_2^{\af} + 0.1\af(-x_1^{\af} + x_2^{\af})) 
                    \},         \\
  y_1^{\af} = 3 + 0.9 \{
          (0.6y_1^{\af} + 0.4y_2^{\af} + 0.1\af( y_1^{\af} - y_2^{\af})) \\
          \hspace{9em} \vee
          (0.8y_1^{\af} + 0.2y_2^{\af} + 0.1\af(-y_1^{\af} + y_2^{\af})) 
                    \},         \\
  x_2^{\af} = -1 + 0.9 \{
          (0.5x_1^{\af} + 0.5x_2^{\af} + 0.1\af( x_2^{\af} - x_1^{\af})) \\
          \hspace{9em} \wedge
          (0.3x_1^{\af} + 0.7x_2^{\af} + 0.1\af(-x_1^{\af} + x_2^{\af})) 
                    \},         \\
  y_2^{\af} = -1 + 0.9 \{
          (0.5y_1^{\af} + 0.5y_2^{\af} + 0.1\af( y_2^{\af} - y_1^{\af})) \\
          \hspace{9em} \vee
          (0.3y_1^{\af} + 0.7y_2^{\af} + 0.1\af(-y_1^{\af} + y_2^{\af})) 
                    \}.
\end{array}
$$
After a simple calculation, we find
$$
\T{\psi}(f_1)_\af =\Big(
  [\frac{750+360\af}{73}, \frac{1470-360\af}{73}], 
  [\frac{1350+360\af}{73},\frac{1070-360\af}{73}], 
                    \Big)', 
$$
which leads to 

\vspace*{-1em}
$$
\T{\psi}(f_1) =\Big(
 (\frac{750}{73}/ \frac{1110}{73}/ \frac{1470}{73}), 
 (\frac{350}{73}/ \frac{710}{73}/  \frac{1070}{73}) 
                   \Big)'. 
$$


%%%%%%%%%%%%%%%%%%%%%%%%
%      Section 5       %
%%%%%%%%%%%%%%%%%%%%%%%%
\section{Pareto optimality} % -----------------------------------------
Here, we confine our attention to the class of stationary policies, 
which simplifies our discussion in the sequel. 
A policy $f^* \in \Pi_F$ is called Pareto optimal 
if there does not exist $f \in \Pi_F$ such that 
$\T{\psi}(f^*) \prec  \T{\psi}(f)$.
In this section, 
we derive the optimal inclusion, 
by which  Pareto optimal policies are characterized. 

The following important result is crucial to the development 
in the characterization of Pareto optimality. 

%\bigskip
\noindent
{\bf Lemma 5.1.} \ 
{\sl 
For any $f, g \in F$, suppose that 

\vspace*{-1.5em}
$$
  \T{\psi}(f) \quad \Big\{ {\inc \atop \prec} \Big\} \quad 
  U_g \T{\psi}(f). 
\eqno{(5.1)}
$$

\vspace*{-1em}
Then, it holds that 

\vspace*{-1.5em}
$$
  \T{\psi}(f) \quad \Big\{ {\inc \atop \prec} \Big\} \quad 
  \T{\psi}(g). 
\eqno{(5.2)}
$$
}
%
%\vspace*{-0.5em}
\Pf
Suppose that $\T{\psi}(f) \ \Big\{ {\inc \atop \prec} \Big\} \ U_g \T{\psi}(f)$.  
Then, we have from Lemma 4.2 {\rm (ii)} that 

$$
  \T{\psi}(f) \quad 
  \Big\{ {\inc \atop \prec} \Big\} \quad 
  U_g \T{\psi}(f) \quad 
  \inc \quad 
  U_g^l \T{\psi}(f) \ \ 
  (l \ge 2),
$$
So, taking the limit in the above as $l \to \infty$, 
(5.2) follows from Lemma 3.2. 
\hfill \qed

%\bigskip

Let $\BS{D}$ be an arbitrary subset of $\C{F}_c(\BR_+)^n$. 
A point $\T{\BS{u}} \in \BS{D}$ is called an efficient element of $\BS{D}$ 
with respect to $\ \inc \ $ on $\C{F}_c(\BR_+)^n$ if and only if 
it holds that there does not exist $\T{\BS{v}} \in \BS{D}$ such that 
$\T{\BS{u}} \prec \T{\BS{v}}$.
We denote by ${\rm eff}(\BS{D})$ 
the set of all elements of $\BS{D}$ efficient with respect to 
$\ \inc \ $ on $\C{F}_c(\BR_+)^n$.
For any $\T{\BS{u}} \in \C{F}_c(\BR_+)^n$, let
$
\BS{U}(\T{\BS{u}}) := \mbox{eff} ( \{ U_f \T{\BS{u}} \mid f \in F \}).
$
Note that $\BS{U}(\T{\BS{u}}) \subset \C{F}_c(\BR_+)^n$. 

Here, we consider the following fuzzy inclusion 
including efficient set-functions $\BS{U}(\cdot)$ on $\C{F}_c(\BR_+)^n$:
$$
   \, \, \T{\BS{u}} \in \BS{U}(\T{\BS{u}}), \quad  
   \T{\BS{u}} \in \C{F}_c(\BR_+)^n. \, \, 
\eqno{(5.3)}
$$
The inclusion of (5.3) is called an optimality equation, 
by which Pareto optimal policies are characterized. 
A solution of (5.3), $\T{\BS{u}}$, is called maximal 
if there does not exist any solution $\T{\BS{u}}'$ of (5.3) 
such that $\T{\BS{u}} \prec \T{\BS{u}}'$. 
Pareto optimal policies are characterized by maximal solutions 
of the optimality equation (5.3). 

%\bigskip
\noindent
{\bf Theorem 5.1.}\ 
{\sl 
A policy $f$ is Pareto optimal if and only if 
a fixed point of the corresponding $U_f$, $\T{\psi}(f)$, 
is a maximal solution to the optimal inclusion (5.3). 
}
%
%\medskip
\Pf
The proof of \lq\lq only if \rq\rq part is easily obtained from Lemma 5.1. 
In order to prove \lq\lq if \rq\rq part, 
suppose that $\T{\psi}(f)$ is a maximal solution of (5.3) 
but $f$ is not Pareto optimal. 
Then, there exists 
$f^{(1)} \in F$ such that $\T{\psi}(f) \prec \T{\psi}(f^{(1)})$. 
Now, suppose that 
$\T{\psi}(f^{(1)}) \not\in \mbox{eff}(\T{\psi}(f^{(1)}))$. 
This assumption assures that there exists 
$f^{(2)} \in F$ 
satisfying $\T{\psi}(f^{(1)}) \prec U_{f^{(2)}}\T{\psi}(f^{(1)})$, 
which implies from (5.1) that 
$\T{\psi}(f^{(1)}) \prec \T{\psi}(f^{(2)})$. 
By repeating this method successively, 
we come to the conclusion that there exists $l$ such that
$f^{(l)} \in F$ 
such that 
$\T{\psi}(f) \prec \T{\psi}(f^{(l)})$ and 
$\T{\psi}(f^{(l)})$ satisfies (5.3), 
which contradicts that $\T{\psi}(f)$ is maximal, as required.
\hfill \qed

%\bigskip
\noindent
{\bf Remark.} \ 
For vector-valued discounted MDPs, 
Furukawa\cite{Fur80} and White\cite{Whi82} 
had derived the optimality equation 
including efficient set-function on $\BR^n$, 
by that Pareto optimal policies are characterized. 
The form of the optimal inclusion (5.3) is corresponding 
to a fuzzy version of MDPs. 


\noindent
{\bf Example 2.} \ 
For the machine maintenance problem of Example 1 given in Section 4, 
Pareto optimal policy is calculated by Theorem 5.1.
From Example 1, we find that 
$$
U_{f_2}\T{\psi}(f_1) = \Big( 
 (\frac{750}{73}/ \frac{1110}{73}/ \frac{1470}{73}), 
 (\frac{349}{73}/ \frac{709}{73}/  \frac{1069}{73}) 
                   \Big)', 
$$

\vspace*{-1em}
Recall that 

\vspace*{-1em}
$$
U_{f_1}\T{\psi}(f_1) = \T{\psi}(f_1) =\Big(
 (\frac{750}{73}/ \frac{1110}{73}/ \frac{1470}{73}), 
 (\frac{350}{73}/ \frac{710}{73}/  \frac{1070}{73}) 
                   \Big)', 
$$
which satisfies 
$U_{f_2}\T{\psi}(f_1) \prec \T{\psi}(f_1)$.
Thus, 
$\T{\psi}(f_1) \in {\rm eff}( \{ U_f\T{\psi}(f_1) \mid f \in F)$,
so that $f_1$ is Pareto optimal from Theorem 5.1. 
In fact, we can find, by solving (4.4) for $f_2$, that 
$$
\T{\psi}(f_2) =\Big(
 (\frac{930}{91}/ \frac{1380}{91}/ \frac{1830}{91}), 
 (\frac{430}{91}/ \frac{880}{91}/  \frac{1330}{91}) 
                   \Big)', 
\mbox{\ and \ }
\T{\psi}(f_2) \prec \T{\psi}(f_1).
$$


% \newpage
%------------------------- References  ---------------------------------------
%\begin{small}
\begin{thebibliography}{99}
%\baselineskip 1mm
%1
\bibitem{Bla62} Blackwell, D., Discrete dynamic programming,
{\it Ann. Math. Stat.} {\bf 33} (1962), 719-726.

%2
\bibitem{Fur80} Furukawa, N., 
Characterization of optimal policies in vector-valued Markovian 
decision processes,
{\it Math. Oper. Res.} {\bf 5} (1980), 271-279.

%3
\bibitem{HarSen94} Hartfiel, D. J. and Seneta, E.
On the theory of Markov Set-chains,
{\it Adv. Appl. Prob.} {\bf 26} (1994), 947-964.

%4
\bibitem{Har98} Hartfiel, D. J., 
{\it Markov Set-chains}, 
(1998), Springer-Verlag, Berlin.

%5
\bibitem{Hin70} Hinderer, K., 
{\it Foundations of Non-Stationary Dynamic Programming with 
Discrete Time Parameter,} 
(1970), Springer-Verlag, New York.

%6
\bibitem{HosKur} Hosaka, M. and Kurano, M., 
Non-discounted Optimal policy in controlled Markov Set-chains,
{\it J. Opern. Res. of Japan}, {\bf 42} (1999), 256-267.

%7
\bibitem{How60} Howard, R.,  
{\it Dynamic Programming and Markov processes}
(1960), MIT Press, Cambridge MA.

%8
\bibitem{KBC87} Kruse, R., Buck-emden, R. and Cordes, R., 
Processor power considerations - an application to fuzzy Markov 
chains,
{\it Fuzzy Sets and Systems}, {\bf 21} (1987), 289-299.

%9
\bibitem{KYNY96} Kurano, M., Yasuda, M., Nakagami, J. and 
Yoshida, Y., 
Markov-type fuzzy decision processes with a discounted reward 
on a closed interval,
{\it European J. O. R.}, {\bf 92} (1996), 649-662.

%10
\bibitem{KSHH98} Kurano, M., Song, J., Hosaka, M. and Huang, Y.,
Controlled Markov Set-Chains with Discounting,
{\it J. Appl. Prob.}, {\bf 35} (1998), 293-302.

%11
\bibitem{KYN99} Kurano, M., Yasuda, M. and Nakagami, J.,
Interval methods for uncertain Markov decision processes, 
{\it in Markov Processes and Controlled Markov Chains}, 
edited by H. Zhenting, J. A. Filar and A. Chen, Kluwer, 
Dordrecht, The Netherlands, (2002), 223-232. 

%12
\bibitem{Kra66} Kuratowski, K, 
{\it Topology}, 
(1966), Academic Press, New York. 

%13
\bibitem{MinOsa70} Mine, H. and Osaki, S., 
{\it Markov Decision Processes},
(1970), Elsevier, Amsterdam. 

%14
\bibitem{Nen84} Nenmaier, A., 
New techniques for the analysis of linear interval equations,
{\it Linear Algebra Appli.}, {\bf 58} (1984), 273-325.

%15
\bibitem{Nov89} Nov{\'a}k, V., 
{\it Fuzzy sets and their applications}, 
(1989), Adam Hilger, Bristole-Boston. 

%16
\bibitem{Put94} Puterman, M. L., 
{\it Markov decision processes: Discrete Stochastic
Dynamic Programming}, (1994), John Wiley \& Sons, INC.

%17
\bibitem{Whi82} White, D. J., 
Multi-objective infinite-horizon discounted Markov Decision 
Processes,
{\it J. Math. Anal. Appl.},  {\bf 89} (1982), 639-647.

%18
\bibitem{Yos98} Yoshida, Y., 
A time-average fuzzy reward criterion in fuzzy decision 
processes,
{\it Information Sciences}, {\bf 110} (1998), 103-112.

%19
\bibitem{Zad65} Zadeh, L. A., 
Fuzzy sets, 
{\it Inform. and Control}, {\bf 8} (1965), 338-353.

\end{thebibliography}


\end{document} %---------------------------------------------------------------

