% latex2e
% mdpfr.tex
% naca2001
% Markov decision processes with fuzzy rewards
% For Special Issues of JNCA

\documentclass[leqno,11pt]{article}

\usepackage{amsmath}  % amstex.sty \boldsymbol
\usepackage{amssymb}  % \mathfrak \mathbb
\usepackage{latexsym} % \Box
\usepackage{graphicx}

\topmargin = 0cm
\oddsidemargin = 0cm  \evensidemargin = 0cm 
\textheight = 50truepc    \textwidth = 33truepc 
% \renewcommand{\baselinestretch}{1.2} %1.2 space

%----------------------Definition----------------------------------------------
\def\T#1{{\widetilde{#1}}}
\def\C#1{{\mathcal{#1}}}

\def\BB#1{{\mathbb{#1}}}
\def\BS#1{{\boldsymbol{#1}}}

\def\U#1{{\underline{#1}}}
\def\O#1{{\overline{#1}}}

\newcommand{\BR}{{\mathbb{R}}}

\newcommand{\af}{{\alpha}}
\newcommand{\ld}{{\lambda}}
\newcommand{\inc}{{\preccurlyeq}}
\newcommand{\Pf}{\noindent {\bf Proof.} \ }
\newcommand{\qed}{{\quad $\square$}}

% -------------------- maketitle ----------------------------------------------
\title
{\bf Markov decision processes with fuzzy rewards} 

\author
{
Masami Kurano${}^{a}$, \ 
Masami Yasuda${}^{b}$  \\
Jun-ichi Nakagami${}^{b}$, \ 
% \thanks{
% Corresponding author: Jun-ichi Nakagami. 
% Tel: +81-43-290-2718.  Fax: +81-43-290-2733.
% {\it E-mail:} nakagami@math.s.chiba-u.ac.jp.
%       }\ \ 
% and \ 
Yuji Yoshida${}^{c}$
}
\date
{\small \it
${}^{a}$ Department of Mathematics, Faculty of Education \\
{\rm E-mail:}{\tt kurano@math.e.chiba-u.ac.jp} \\
${}^{b}$ Department of Mathematics and Informatics, Faculty of Science \\ 
{\rm E-mail:}{\tt yasuda@math.s.chiba-u.ac.jp}, 
             {\tt nakagami@math.s.chiba-u.ac.jp} \\
         Chiba University, Chiba 263-8522 Japan \\ 
${}^{c}$ Faculty of Economics and Business Administration \\
    The University of Kitakyushu, Kitakyushu 802-8577 Japan \\
{\rm E-mail:}{\tt yoshida@kitakyu-u.ac.jp} 
}

\begin{document} %------------------------------------------------------------
\maketitle % -----------------------------------------------------------------

\vspace*{-1em}

\begin{abstract}
In this paper, 
we consider the model 
that the information on the rewards 
in vector-valued Markov decision processes 
includes imprecision or ambiguity.
The fuzzy reward model is analyzed as follows:
The fuzzy reward is represented by 
the fuzzy set on the multi-dimensional Euclidian space $\BR^p$
and 
the infinite horizon fuzzy expected discounted reward(FEDR) 
from any stationary policy 
is characterized 
as a unique fixed point of the corresponding contractive operator.
Also,
we fined a Pareto optimal policy 
which maximizes the infinite horizon FEDR 
over all stationary policies 
under the pseudo order 
induced by a convex cone $\BR^p$.
As a numerical example, the machine maintenance problem is considered.
\end{abstract}

%\noindent
\textit{Keywords}: 
Multi-dimensional fuzzy reward model, Markov decision process, 

\hspace*{4.5em}
Pareto optimal, fuzzy optimality equation.

%\noindent
\textit{AMS 1991 subject classification}. \ 
Primary: 90c40; Secondary: 90c39.

%------------------------------------------------------------------------------
\section*{1. Introduction} 
In mathematical modeling in terms of Markov decision processes
(MDPs, in short, cf.~\cite{Bla62, How60, MinOsa70, Put94}),
it often occurs that the information on the reward function 
includes imprecision or ambiguity. 
As an example, 
the reward earned in a day is about 700 dollars 
or closed to 800 dollars. 
On the other hand, 
multi-criteria decision making is typically involving 
flexible requirements for the optimality.
In order to deal with uncertain data and flexible requirements 
we can use a fuzzy set representation (cf.~\cite{Sto98}).
In this paper,
we consider the case that the $\BR^p$-valued rewards 
in standard MDPs are specified by fuzzy sets on $\BR^p$, 
where $\BR^p$ is a $p$-dimensional Euclidean space ($p \ge 1$).

Recently, Kurano et al \cite{KYNY00a} has introduced 
a pseudo order \ $\inc_K$ \ 
in the class of fuzzy sets on $\BR^p$, 
which is a natural extension of fuzzy max order 
(cf.\cite{Fur97, RR85}) in fuzzy numbers on $\BR$ 
and induced by a convex cone $K$ in $\BR^p$.
Under this pseudo order \ $\inc_K$, 
we fined a Pareto optimal policy which maximizes 
the infinite horizon fuzzy expected discounted reward (FEDR) 
over all stationary policies.
Associated with each stationary policy is 
a corresponding contractive operator on fuzzy sets, 
whose fixed point represents the infinite horizon FEDR.  
Moreover, the Pareto optimal policies are characterized by 
maximal solutions of an optimal equation 
including efficient fuzzy set functions. 
As a numerical example, the machine maintenance problem is considered. 
For an interval or fuzzy treatment for MDPs 
with uncertain transition matrices, 
see \cite{KSHH98, KYN99, KYNY00b} 
in which the intervals or fuzzy sets are used 
to describe uncertain transition matrices.
Also, for the optimization of fuzzy dynamic system refer to 
\cite{KYNY96, Yos98}.

This paper is organized as follows: 
In Section 2, 
we shall give some notations needed for fuzzy treatments 
and a pseudo order relation 
of fuzzy sets on $\BR^p$ 
is reviewed referring to Kurano et al \cite{KYNY00a}
and the expectation of discrete fuzzy random variables is specified.
In Section 3, 
we describe the fuzzy reward model 
and specify the optimization problem. 
In Section 4, 
the infinite horizon FEDR from a stationary policy is given 
as a fixed point of a corresponding operator, 
which is used to obtain the optimality equation 
and characterize a Pareto optimal policy in Section 5.

%---------------------------------------------------------------------------
\section*{2. Preliminaries} 
We write fuzzy sets on $\BR^p$ by their membership functions 
$\T{s} : {\BR^p} \to [0,1]$ 
(see Nov\'{a}k~\cite{Nov89} and Zadeh~\cite{Zad65}).
The $\af$-cut ($\af \in [0,1]$) of the fuzzy set $\T{s}$ on $\BR^p$ 
is defined as 
$$
\T{s}_{\af} := \{ x \in \BR^p \mid \T{s}(x) \geq \af \}
            \ (\af > 0) \quad \mbox{and} \quad
\T{s}_{0} := \mbox{\rm cl} \{ x \in {\BR^p} \mid \T{s}(x) > 0 \},
$$
where {\rm cl} denotes the closure of the set.
A fuzzy set $\T{s}$ is called convex if
$$
\T{s}(\ld x + (1- \ld )y) \ge \T{s}(x) \wedge \T{s}(y)
\quad x,y \in {\BR^p}, \ \ld \in [0,1],
$$
where $a \wedge b = \min\{a,b\}$.

Note that $\T{s}$ is convex if and only if the $\af$-cut $\T{s}_{\af}$ is 
a convex set for all $\af \in [0,1]$.
Let $\C{F}({\BR^p})$ be the set of all convex fuzzy sets 
whose membership functions $\T{s} : {\BR^p} \to [0,1]$
are upper-semicontinuous and 
normal ($\sup_{x \in \BR^p} \T{s}(x) = 1$) 
and have a compact support. 
In the one-dimensional case $p=1$, 
$\C{F}(\BR)$ denotes the set of all fuzzy numbers.
Let $\C{C}(\BR^p)$ be the set of all compact convex subsets of $\BR^p$. 
We note that when $p=1$, $\C{C}(\BR)$ denotes 
the set of bounded and closed intervals in $\BR$.


The definitions of addition and scalar multiplication on $\C{F}(\BR^p)$ 
are as follows: 
For $\T{s}, \T{r} \in \C{F}(\BR^p)$ and $\ld \geq 0$,
$$
(\T{s} + \T{r})(x) :=
  \sup_{\scriptstyle{x_1, x_2 \in \BR^p}\atop {\ x_1 + x_2 = x}}
     \{ \T{s}(x_1) \wedge \T{r}(x_2) \} ,
\leqno{(2.1)}
$$
$$
  ( \ld \T{s})(x) :=
     \left\{
       \begin{array}{ll}
        \T{s}(x/\ld) & \mbox{if} \ \ld>0 \\
        {\bf 1}_{\{0\}}(x) & \mbox{if} \ \ld=0
       \end{array}
       \quad (x \in \BR^p),
     \right.
\leqno{(2.2)}
$$
where ${\bf 1}_{\{\cdot\}}(\cdot)$ is an indicator. 
By using set operations $A + B := \{ x+y \mid x \in A, y \in B\}$ 
and $\ld A := \{ \ld x \mid x \in A \}$ 
for any non-empty sets $A, B \subset \BR^p$, 
the following holds immediately.
$$
(\T{s} + \T{r})_{\af} = \T{s}_{\af} + \T{r}_{\af} 
\quad \mbox{and} \quad 
(\ld \T{s})_{\af} = \ld \T{s}_{\af} 
\quad (\af \in [0,1]).
\leqno{(2.3)}
$$


Let $\rho$ be the Hausdorff metric on $\C{C}(\BR^p)$, 
that is, 
for $A, B \in \C{C}(\BR^p)$, 
$$
\rho(A,B) = \max \{ \max_{a \in A}d(a,B), \max_{b \in B}d(b,A) \},
$$
where $d$ is a metric in $\BR^p$ and 
$\displaystyle{
d(x,D) = \min_{y \in D}d(x,y)
}$  
for $x \in \BR^p$ and $D \in \C{C}(\BR^p)$.
Extending this $\rho$ to  $\C{F}(\BR^p)$, 
we define, with abuse of notation, 
the Hausdorff metric on $\C{F}(\BR^p)$ by
$$
\rho(\T{u},\T{v}) = \sup_{\af \in [0,1]}
\rho(\T{u}_{\af},\T{v}_{\af}) \quad \mbox{for \ }
\T{u},\T{v} \in \C{F}(\BR^p),
\leqno{(2.4)}
$$
where $\T{u}_{\af}$ and $\T{v}_{\af}$ are $\af$-cuts of 
$\T{u}$ and $\T{v}$ respectively.

Then, the following facts are well known.

\bigskip
\noindent
{\bf Lemma 2.1} (cf. \cite{PR86}). 
\quad
{\sl 
The metric space $(\C{F}(\BR^p), \rho)$ is complete.
}

\bigskip
\noindent
{\bf Lemma 2.2} (cf. \cite{DK94}). 
\quad
{\sl 
If $\T{u}, \T{v}, \T{u'}, \T{v'}$ and $\T{r} \in \C{F}(\BR^p)$, then

\vspace*{-0.5em}
\begin{itemize}
\item[\rm{(i)}]
$\rho(\ld \T{u}, \ld \T{v}) = \ld \rho(\T{u},\T{v})$ \ 
for all $\ld \ge 0$.

\vspace*{-0.5em}
\item[\rm{(ii)}]
$\rho(\T{u}+\T{u'}, \T{v}+\T{v'}) 
\le \rho(\T{u},\T{v}) + \rho(\T{u'},\T{v'})$,

\vspace*{-0.5em}
\item[\rm{(iii)}]
$\rho(\T{r}+\T{u}, \T{r}+\T{v}) 
= \rho(\T{u},\T{v})$.
\end{itemize}
}

Here, we pick up a pseudo order relation introduced in Kurano et al 
\cite{KYNY00a},
which is necessary for our problem formulation in the sequel.
The partial order relation $\inc_1$ on $\C{C}(\BR)$ 
is defined as follows: 
For any $[c_1,c_2]$, $[c'_1,c'_2] \in \C{C}(\BR)$, 
$[c_1,c_2] \ \inc_1 \ [c'_1,c'_2]$ means that 
$c_1 \le c'_1$ and $c_2 \le c'_2$.

Let $K$ be a non-empty cone of $\BR^p$.
Using this $K$, we can define a pseudo order relation $\inc_K$
on $\BR^p$ by $x \ \inc_K \ y$ if and only if $y-x \in K$. 
By the pseudo order $\inc_K$ on $\BR^p$,
a pseudo order $\inc_K$ on $\C{F}(\BR^p)$ is defined as follows. 

\bigskip
\noindent
{\sl
For $\T{s},\T{r} \in \C{F}(\BR^p)$, 
$\T{s} \ \inc_K \ \T{r}$ means the following {\rm (F.a)} and {\rm (F.b)}:
\begin{itemize}

\vspace*{-0.5em}
\item[{\rm (F.a)}] 
For any $x \in \BR^p$, there exists $y \in \BR^p$ 
such that $x \ \inc_K \ y$ and $\T{s}(x) \leq \T{r}(y)$.

\vspace*{-0.5em}
\item[{\rm (F.b)}] 
For any $y \in \BR^p$, there exists $x \in \BR^p$ 
such that $x \ \inc_K \ y$ and $\T{s}(x) \geq \T{r}(y)$.
\end{itemize}
}
When $p=1$ and $K=[0,\infty)$, 
the $\inc_K$ on $\C{F}(\BR)$ is a partial order 
and called the fuzzy max order (cf. \cite{Fur97, RR85})
defined by $\inc_1$.
That is, for $\T{s}, \T{r} \in \C{F}(\BR)$,
$\T{s}\ \inc_1 \ \T{r}$ means that 
$\T{s}_{\af}^{L} \leq  \T{r}_{\af}^{L}$
and 
$\T{s}_{\af}^{U} \leq  \T{r}_{\af}^{U}$
for all $\af \in [0,1]$, where the $\af$-cuts of 
$\T{s}$ and $\T{r}$ are denoted respectively by 
$\T{s}_{\af} =[\T{s}_{\af}^{L},\T{s}_{\af}^{U}]$ and
$\T{r}_{\af} =[\T{r}_{\af}^{L},\T{r}_{\af}^{U}]$.

Define the dual cone of a cone $K$ by
$$
K^{+} := \{ a \in \BR^p \mid a \cdot x \ge 0 \ 
\mbox{for all} \ x \in K \},
$$
where $x \cdot y$ denotes the inner product on $\BR^p$ 
for $x,y \in \BR^p$.
For a subset $A \subset \BR^p$ and $a \in \BR^p$, 
we define
$$
a \cdot A := \{ a \cdot x \mid x \in A \} \ (\subset \BR).
\leqno{(2.5)}
$$
The definition (2.5) means the projection of $A$ 
on the extended line of the vector $a$ if $a \cdot a = 1$.
It is trivial that $a \cdot A \in \C{C}(\BR)$ 
if $A \in \C{C}(\BR^p)$ and $a \in \BR^p$.

The pseudo order relation $\inc_K$ on $\C{F}(\BR^p)$ 
is characterized by $\inc_1$ on $\C{F}(\BR)$ through the projection (2.5),
where the proof is in \cite{KYNY00a}.

\bigskip
\noindent
{\bf Lemma 2.3} \cite{KYNY00a}. \quad 
{\sl 
Let $\T{u}, \T{v} \in \C{F}(\BR^p)$. Then,
$\T{u} \ \inc_K \ \T{v}$ on $\C{F}(\BR^p)$ 
if and only if 
$a\cdot \T{u}_{\af} \ \inc_1 \ a\cdot \T{v}_{\af}$ on $\C{F}(\BR)$ 
for all $a \in K^+$ and $\af \in [0,1]$.
}

\bigskip
\noindent
{\bf Lemma 2.4} \cite{KYNY00a}. \quad 
{\sl 
Let a sequence $\{ \T{u}_l \} \subset \C{F}(\BR^p)$ 
be such that $\T{u}_1 \ \inc_K \ \T{u}_2 \ \inc_K \ \cdots,$ and 
$\T{u} = \lim_{l \to \infty}\T{u}_l \in \C{F}(\BR^p)$. 
Then, it holds that $\T{u}_1 \ \inc_K \ \T{u}$.
}

\bigskip

The following lemma is used in the sequel.

\bigskip
\noindent
{\bf Lemma 2.5.} \quad 
{\sl Let $A, B \in \C{C}(\BR^p)$ and $a \in \BR^p$. 
Then, we have: 
\begin{itemize}

\vspace*{-0.5em}
\item[{\rm (i)}] 
$a\cdot (A+B) = a\cdot A + a\cdot B$,

\vspace*{-0.5em}
\item[{\rm (ii)}] 
$a\cdot (\ld A) = \ld(a\cdot A)$ \ for all $\ld \ge 0$.
\end{itemize}
}

%\bigskip
\Pf
For $A, B \in \C{C}(\BR^p)$, $a\cdot (A+B) \in \C{C}(\BR)$, 
so that, 
$a\cdot (A+B) = [a\cdot (x^L + y^L), a\cdot (x^U + y^U)]$ 
for some $x^L, x^U \in A$ and $y^L, y^U \in B$.
Since
$a\cdot (x^L + y^L) = a\cdot x^L + a\cdot y^L$, 
$a\cdot x^L \in a\cdot A$ and
$a\cdot y^L \in a\cdot B$, 
it holds 
$a\cdot (x^L + y^L) \in a\cdot A + a\cdot B$. 
Similarly,
$a\cdot (x^U + y^U) \in a\cdot A + a\cdot B$. 
Thus,
$a\cdot (A + B) \subset a\cdot A + a\cdot B$. 
Conversely, if we set 
$a\cdot A = [a\cdot x^L, a\cdot x^U]$ and 
$a\cdot B = [a\cdot y^L, a\cdot y^U]$, 
$a\cdot (A+B) = [a\cdot (x^L + y^L), a\cdot (x^U + y^U)]$, 
which implies 
$a\cdot A + a\cdot B \subset a\cdot (A + B)$.

Also, (ii) clearly holds, as required.
\qed

\bigskip

In order to formulate the optimization problem in the next section,
we need the concept of the expectation of discrete fuzzy random variables.

Let $(\Omega, \C{B}, P)$ be a probability space and 
$\T{X} : \Omega \to \C{F}(\BR^p)$ 
a discrete fuzzy random variable with its range 
$\{ \T{s}_1, \T{s}_2, \cdots, \T{s}_l \} \subset \C{F}(\BR^p)$.
Then, we define the expectation of $\T{X}$ by
$$
E[\T{X}] = \sum_{i=1}^{l} \ \T{s}_i \  P(\T{X} = \T{s}_i).
\leqno{(2.6)}
$$
Note that the expectation in (2.6) is defined in (2.1) and (2.2).
The definition of (2.6) is corresponding to 
the discrete case of the integral of a set-valued function 
(cf. \cite{Aum65}) or 
the expectation of general fuzzy random variables 
(cf. \cite{PR86}). 

The following clearly holds.

\bigskip
\noindent
{\bf Lemma 2.5.} \quad 
{\sl 
If $\T{X}$ and $\T{Y}$ are discrete fuzzy random variables 
whose ranges are finite subsets of $\C{F}(\BR^p)$, 
then
\begin{itemize}

\vspace*{-0.5em}
\item[{\rm (i)}] 
$E[\T{X}] \in \C{F}(\BR^p)$,

\vspace*{-0.5em}
\item[{\rm (ii)}] 
$E[\T{X} + \T{Y}] = E[\T{X}] + E[\T{Y}]$,

\vspace*{-0.5em}
\item[{\rm (iii)}] 
$E[\ld \T{X}] = \ld E[\T{X}]$ \ for all $\ld \ge 0$.
\end{itemize}
}

%---------------------------------------------------------------------------
\section*{3. The fuzzy reward model} 
In this section, 
we formulate MDPs with fuzzy rewards on $\BR^p$ 
and specify our optimization problem.
Let $S$ and $A$ be finite sets denoted by 
$S = \{ 1,2, \cdots, n \}$ and 
$A = \{ 1,2, \cdots, k \}$.
The sequential decision model consists of four objects:
$$
( S, A, \{ q_{ij}(a) ; i, j \in S, \ a \in A \}, \T{r} ),
$$
where $S$ and $A$ denote the state and action spaces respectively 
and $\T{r} = \T{r}(i,a) \in \C{F}(\BR^p)$
is a fuzzy reward function on $S \times A$ 
and $\{ q_{ij}(a) \}$ is the law of motion, i.e., 
for each $(i,a) \in S\times A$, $q_{ij}(a) \ge 0$ and 
$\sum_{j \in S}q_{ij}(a) =1$.

When the system is in state $i \in S$ and 
we take action $a \in A$, 
the present state moves to a new state $j \in S$ 
selected according to the probability distribution $q_{i\cdot}(a)$ 
and we receive a fuzzy reward $\T{r}(i,a) \in \C{F}(\BR^p)$.
This process is then repeated from the new state $j \in S$.
The sample space is the product space $\Omega = (S\times A)^{\infty}$ 
such that the projection 
$X_t$ and $\Delta_t$ on the $t$-th factor $S$ and $A$ 
describe the state and the action at the $t$-th time of the process 
$(t =1, 2, \ldots)$.

We denote by $F$ the set of all functions from $S$ to $A$.
A policy $\pi$ is a sequence $(f_1, f_2, \ldots )$ of functions with 
$f_t \in F \ (t \ge 1)$.
Let $\Pi$ be the class of policies. 
We denote by $f^{\infty}$ the policy $(f_1, f_2, \ldots )$ with 
$f_t = f$ for all $t \ge 1$ and some $f \in F$. 
Such a policy is called stationary and denoted simply by $f \in F$. 
The set of all stationary policies will be denoted by $\Pi_F$.
Then, for each policy $\pi \in \Pi$ and starting state $i \in S$, 
we can define the probability measure $P^i_{\pi}$ on $\Omega$ 
in a usual way. 
Here, we consider the expected fuzzy reward in which 
the future reward is discounted with a factor 
$\beta \ (0<\beta<1)$. 

For any policy $\pi \in \Pi$ and starting state $i \in S$, 
let 
$$
\T{\phi}_{T}(i, \pi) = 
\sum_{t=1}^{T}\beta^{t-1}E_{\pi}^{i}[\T{r}(X_t, \Delta_t)],
\leqno{(3.1)}
$$
where $E_{\pi}^{i}$ is the expectation with respect to 
$P_{\pi}^{i}$ and its expectation of fuzzy random variable 
is defined by (2.6). 
We note from Lemma 2.5 that 
$\T{\phi}_{T}(i, \pi) \in \C{F}(\BR^p)$
for $i \in S$, $\pi \in \Pi$ and $T \ge 1$. 

In order to rewrite (3.1) by using vectors and matrices, 
we shall introduce some notations.
Let
$\C{F}(\BR^p)^n$ be the set of all $n$-dimensional column vectors 
whose elements are in $\C{F}(\BR^p)$, i.e., 
$$
\C{F}(\BR^p)^n := \{ 
\T{u} = ( \T{u}_1, \T{u}_2, \ldots, \T{u}_n )' \mid 
\T{u}_i \in \C{F}(\BR^p), 1 \le i \le n \},
$$
where $d'$ denotes the transpose of a vector $d$. 

The Hausdorff metric $\rho$ on $\C{F}(\BR^p)^n$ is defined 
(with abuse of notation) by 
$$
\rho(\T{u},\T{v}) = \max_{1 \le i \le n} 
\rho(\T{u}_i, \T{v}_i),
$$
where 
$\T{u} = ( \T{u}_1, \T{u}_2, \ldots, \T{u}_n )'$, 
$\T{v} = ( \T{v}_1, \T{v}_2, \ldots, \T{v}_n )'
\in  \C{F}(\BR^p)^n$ and 
$\rho(\T{u}_i, \T{v}_i)$ is defined in (2.4).
Then, from Lemma 2.1, we observe that 
the metric space $(\C{F}(\BR^p)^n, \rho)$ is complete.

For a $n\times n$ stochastic matrix $Q = (q_{ij})$ and 
$\T{u} = ( \T{u}_1, \T{u}_2, \ldots, \T{u}_n )' \in \C{F}(\BR^p)^n$, 
the product 
$Q\T{u} \in \C{F}(\BR^p)^n$ 
will be defined by 
$$
(Q\T{u})_i = \sum_{j=1}^{n} q_{ij}\T{u}_j \ \ 
(1 \le i \le n).
\leqno{(3.2)}
$$
Here, we associate with each $f \in F$ 
the $n$-dimensional column fuzzy vector 
$\T{r}(f) \in \C{F}(\BR^p)^n$
whose $i$-th element is 
$\T{r}(i,f(i)) \in \C{F}(\BR^p)$ 
and the $n\times n$ stochastic matrix $Q(f)$ 
whose $(i,j)$ element is $q_{ij}(f(i))$. 
For each policy $\pi \in \Pi$, let
$$
\T{\phi}_{T}(\pi) = (
\T{\phi}_{T}(1,\pi), \T{\phi}_{T}(2,\pi), \ldots, 
\T{\phi}_{T}(n,\pi) ) \in 
\C{F}(\BR^p)^n
\ \ 
(T \ge 1).
$$
Then, we have the following.


\bigskip
\noindent
{\bf Lemma 3.1.} \quad 
{\sl
For any $\pi = (f_1, f_2, \ldots) \in \Pi$, we have: 
\begin{itemize}

\vspace*{-0.5em}
\item[{\rm (i)}] 
$\T{\phi}_{T}(\pi)$ is described by the following matrix representation. 
\end{itemize}
\vspace*{-0.5em}
$$
\T{\phi}_{T}(\pi) = 
\T{r}(f_1) + \beta Q_1(\pi) \T{r}(f_2) + \cdots +
\beta^{T-1} Q_{T-1}(\pi) \T{r}(f_T)
\ \ (T \ge 1),
\leqno{(3.3)}
$$

\vspace*{-0.5em} \hspace*{4em}
where 
$Q_{t}(\pi) = Q(f_1)\cdots Q(f_t)$ \ $(t \ge 1)$.
\begin{itemize}

\item[{\rm (ii)}] 
$ \{ \T{\phi}_{T}(\pi) \}_{T=1}^{\infty} $ 
is a Cauchy sequence.
\end{itemize}
}

\Pf
By the definition, 
for any $t \ge 0$ we have that
$$
E_{\pi}^{i} [\T{r}(X_t,\Delta_t)]= 
\sum_{j \in S} P_{\pi}^{i}(X_t = j) \T{r}(j,f_t(j)) = 
\sum_{j \in S} Q_{t}(\pi)_{ij} \T{r}(j,f_t(j)), 
$$
which clearly leads to (3.3).

For any $T > H$, it holds from Lemma 2.2 that
$$
\rho( \T{\phi}_{T}(\pi), \T{\phi}_{H}(\pi) ) 
\le 
\rho( \T{0}, \sum_{t=H+1}^{T} \beta^{t-1} Q_{t-1}(\pi) \T{r}(f_t) )
\quad \quad \quad 
$$

\vspace*{-1em}
$$
\quad \quad \quad = 
\beta^{H} \rho( \T{0}, \sum_{t=H+1}^{T} \beta^{t-H-1} Q_{t}(\pi) \T{r}(f_t) )
\le 
\beta^{H} \rho ( \T{0}, \T{r} )/(1-\beta),
$$
where $\T{0} \equiv {\bf 1}_{\{0\}}$. This implies (ii), as required. 
\qed

\bigskip

By Lemma 3.1, the infinite horizon FEDR from $\pi$ can be defined by
$$
\T{\phi}(\pi) = \lim_{T \to \infty} \T{\phi}_{T}(\pi).
$$

In order to specify our optimization problem, 
we extend the pseudo order $\inc_K$ on $\C{F}(\BR^p)$
given in the preceding section to that on $\C{F}(\BR^p)^n$
as follows: 
For 
$\T{u} = ( \T{u}_1, \T{u}_2, \ldots, \T{u}_n )'$, 
$\T{v} = ( \T{v}_1, \T{v}_2, \ldots, \T{v}_n )' \in  \C{F}(\BR^p)^n$, 
$\T{u} \ \inc_K \ \T{v}$ means 
$\T{u}_i \ \inc_K \ \T{v}_i$ for all $i \ (1 \le i \le n)$.

Then, our problem is to maximize the 
$\T{\phi}(\pi) \in \C{F}(\BR^p)^n$ 
over all policies $\pi \in \Pi$ 
with respect to the pseudo order $\inc_K$.

%---------------------------------------------------------------------------
\section*{4. Stationary policies and operators} 
In this section, 
the infinite horizon FEDR from a stationary policy 
is given as a unique fixed point of a corresponding operator. 
Associated with each $f \in F$ is a corresponding operator 
$U_f : \C{F}(\BR^p)^n \to \C{F}(\BR^p)^n$ 
defined as follows: 
For $\T{u} \in \C{F}(\BR^p)^n$,
$$
U_f \T{u} = \T{r}(f) + \beta Q(f) \T{u},
\leqno{(4.1)}
$$
where the arithmetics in (4.1) are defined in the preceding sections.

Since it holds that 
$\ld (\T{c} + \T{d}) = \ld \T{c} + \ld \T{d}$ 
for any $\T{c}, \T{d} \in \C{F}(\BR^p)$ and $\ld \ge 0$, 
the following lemma is easily proved.

\bigskip
\noindent
{\bf Lemma 4.1.} \quad 
{\sl 
If $Q$ is $n\times n$ stochastic matrix and 
$\T{u}, \T{v} \in \C{F}(\BR^p)^n$, 
then it holds that 
}
$$
Q(\T{u} + \T{v}) = Q\T{u} + Q\T{v}.
$$

For any policy $\pi = (f_1, f_2, \ldots)$, 
let $\pi^{-l} = (f_{l+1}, f_{l+2}, \ldots)$ 
for each $l \ge 1$. 
The sequence $\{ \T{\phi}_{T}(\pi) \}_{T=1}^{\infty}$ 
is recursively described.

\bigskip
\noindent
{\bf Lemma 4.2.} \quad 
{\sl 
For any policy $\pi = (f_1, f_2, \ldots)$, we have 
$$
\T{\phi}_{T}(\pi) = 
U_{f_1} U_{f_2} \cdots U_{f_l} \T{\phi}_{T-l}(\pi^{-l}) 
\quad \mbox{for each \ } l \ge 1.
\leqno{(4.2)}
$$
}

\Pf
Since 
$\T{\phi}_{1}(\pi^{-1}) = \T{r}(f_2)$, we have
$$
\T{\phi}_{2}(\pi) = 
\T{r}(f_1) + \beta Q(f_1)\T{r}(f_2) = 
U_{f_1} \T{\phi}_{1}(\pi).
$$
For $T=3$, from Lemma 4.1, we have that 
$$
\T{\phi}_{3}(\pi) = 
\T{r}(f_1) + \beta Q(f_1)\T{r}(f_2) 
+ \beta^2 Q(f_1)Q(f_2)\T{r}(f_3)
\quad \quad \quad 
$$

\vspace*{-1em}
$$
\quad \quad \quad \quad 
= 
\T{r}(f_1) + \beta Q(f_1)
\big( 
\T{r}(f_2) + \beta Q(f_2)\T{r}(f_3) 
\big)
= 
U_{f_1}\T{\phi}_2(\pi^{-1}).
$$
By induction on $T$ and $l$, 
we can easily prove (4.2). 
\qed

\bigskip

Here are some basic properties of $U_f$. 
The following lemma is easily proved from Lemma 2.2.

\bigskip
\noindent
{\bf Lemma 4.3.} \quad 
{\sl
For $f \in F$, $U_f$ is a contraction 
with modulus $\beta$, i.e., 
$$
\rho(U_f\T{u}, U_f\T{v}) 
\le 
\beta \rho(\T{u}, \T{v}), 
\quad 
\mbox{for \ }
\T{u}, \T{v} \in \C{F}(\BR^p)^n.
$$
}

\vspace*{-1em}

\noindent
{\bf Lemma 4.4.} \quad 
{\sl
Let $K$ be a convex cone of $\BR^p$. 
Then, for $f \in F$, $U_f$ is monotone with respect to 
the pseudo order $\inc_K$ on $\C{F}(\BR^p)^n$, i.e., 
for any $\T{u}, \T{v} \in \C{F}(\BR^p)^n$ with 
$\T{u} \ \inc_K \ \T{v}$, it holds that 
$
U_{f}\T{u} \ \inc_K \ U_{f}\T{v}.
$
}

\bigskip
\Pf
>From Lemma 2.3, it suffices to show that 
$
a\cdot (U_{f}\T{u})_{i,\af} 
\ \inc_1 \ 
a\cdot (U_{f}\T{v})_{i,\af}
$
for all $a \in K$, $\af \in [0,1]$ and $i=1,2,\ldots,n$,
where $(U_{f}\T{v})_{i,\af}$ is the $\af$-cut of the
$i$-th element of $U_{f}\T{v}$. 
Applying Lemma 2.5, we get 
$$
a\cdot (U_{f}\T{v})_{i,\af} = 
a\cdot \T{r}(i,f(i))_{\af} + 
\beta \sum_{j=1}^{n} q_{ij}(f(i))(a\cdot \T{u}_{j,\af}).
\leqno{(4.2)}
$$
Since $\T{u} \ \inc_K \ \T{v}$ implies from Lemma 2.3 that 
$a\cdot \T{u}_{j,\af} \ \inc_1 \ a\cdot \T{v}_{j,\af}$
for all $j=1,2,\ldots,n$, 
(4.2) implies that 
$a\cdot (U_f \T{u})_{j,\af} \ \inc_1 \ a\cdot (U_f \T{v})_{j,\af}$.
This completes the proof.
\qed

\bigskip
By Lemma 4.2,
$\T{\phi}_T(f) = U_f \T{\phi}_{T-1}(f)$ for all $T \ge 2$. 
As $T \to \infty$ in the above, 
$\T{\phi}(f)$ is a fixed point of $U_f$. 
Thus, noting Lemma 4.3, the characterization of 
$\T{\phi}(f)$ is immediately formulated as a theorem.

\bigskip
\noindent
{\bf Theorem 4.1.} \quad 
{\sl
For any stationary policy $f \in F$, 
$\T{\phi}(f)$ is a unique solution of the 
following equation: 
$$
\T{u} = U_f \T{u}, \quad 
\T{u} \in \C{F}(\BR^p)^n.
\leqno{(4.3)}
$$
}

\bigskip


Note that (4.3) can be rewritten as the $\af$-cut equation:
$$
\T{u}_{\af} = \T{r}(f)_{\af} 
+ \beta Q(f) \T{u}_{\af}, \quad 
\af \in [0,1],
\leqno{(4.4)}
$$
where
$\T{u}_{\af} = 
( \T{u}_{1,\af}, \ldots, \T{u}_{n,\af} )'$ 
and 
$\T{r}(f)_{\af} = ( 
\T{r}(1,f(1))_{\af}, \ldots, 
\T{r}(n,f(n))_{\af} )' \in \C{C}(\BR^p)^n$.

\bigskip

>From a contraction of $U_f$, the next corollary holds.

\bigskip
\noindent
{\bf Corollary 4.1.} \quad 
{\sl 
For any stationary policy $f \in F$, 
$$
\T{\phi}(f) = \lim_{l \to \infty} 
U_{f}^{l}\T{u} \quad 
(\T{u} \in \C{F}(\BR^p)^n).
$$
}

As a simple example, we consider a fuzzy treatment 
for a machine maintenance problem dealt with in 
(\cite{MinOsa70}, p.1, p.17-18).

\bigskip

\noindent
{\bf a machine maintenance problem}. \ 
A machine can be operated synchronously, say, once an hour. 
At each period there are two states; 
one is operating(state 1), and the other is in failure(state 2).
If the machine fails, it can be restored to perfect functioning by repair. 
At each period, if the machine is running, 
we earn the fuzzy return of (2,3,4) dollars per period; 
the probability of being in state 1 at the next step is 
$0.7$ 
and the probability of moving to state 2 is 
$0.3$ 
where for any $a < b < c$, 
the fuzzy number $(a, b, c)$ on $\BR$ is defined by
$$
(a, b, c)(x) = 
  \left\{
  \begin{array}{ll}
   (x-a)/(b-a) \vee 0 & \mbox{if \ } x \le b, \\
   (x-c)/(b-c) \vee 0 & \mbox{if \ } b \le x.
  \end{array}
  \right.
$$
If the machine is in failure, 
we have two actions to repair the failed machine; 
one is a usual repair, denoted by 1, 
that yields the fuzzy reward $(-2,-1,0)$ dollars
with the probability $0.4$ moving in state 1 
and the probability $0.6$ being in state 2;  
another is a rapid repair, denoted by 2, 
that requires the fuzzy reward $(-3,-2,-1)$ dollars
with the probability $0.6$ moving in state 1 
and the probability $0.4$ being in state 2. 

For the model considered, $S = \{ 1, 2 \}$ 
and there exist two stationary policies, $F = \{ f_1, f_2 \}$ 
with $f_1(2) = 1$ and $f_2(2) =2$, 
where $f_1$ denotes a policy of the usual repair and 
$f_2$ a policy of the rapid repair.
The state transition diagrams 
and fuzzy reward vector
for two policies are shown in Figure 1.


% Fig.1 ---------------------------------------------------------------------
%\input fa.txt
\vspace*{-2.5cm} 

\begin{figure}[htbp]
\unitlength 0.001in
%\hspace*{2.5cm}
\begin{picture}(50,100)
% CIRCLE 2 0 3 0
% 4 100 180 114 182 115 180 115 180
%
\special{pn 8}%
\special{ar 1000 1400 141 141  0.0000000 6.2831853}%
% CIRCLE 2 0 3 0
% 4 260 180 274 182 275 180 275 180
%
\special{pn 8}%
\special{ar 2600 1400 141 141  0.0000000 6.2831853}%
% ELLIPSE 2 0 3 0
% 4 180 181 102 204 109 191 250 190
%
\special{pn 8}%
\special{ar 1800 1410 780 230  0.4168981 2.6949888}%
% SPLINE2 2 0 3 0
% 42 1087 1697 1106 1685 1127 1674 1151 1662 1177 1652 1204 1642 1234 1632	1265 1623 1299 1614 1333 1606 1370 1598 1407 1591 1446 1585 1487 1579 1528	1574 1570 1570 1613 1567 1656 1564 1700 1562 1744 1561 1788 1560 1833 1560	1877 1561 1921 1563 1965 1565 2008 1568 2050 1572 2092 1577 2133 1582 2172	1588 2211 1594 2248 1602 2283 1610 2318 1618 2350 1627 2381 1636 2409 1646	2436 1657 2461 1668 2483 1679 2503 1691 2503 1691
%
\special{pn 8}%
\special{pa 1087 1297}%
\special{pa 1114 1280}%
\special{pa 1143 1266}%
\special{pa 1172 1254}%
\special{pa 1202 1243}%
\special{pa 1233 1232}%
\special{pa 1263 1223}%
\special{pa 1294 1215}%
\special{pa 1326 1208}%
\special{pa 1357 1201}%
\special{pa 1388 1194}%
\special{pa 1420 1189}%
\special{pa 1451 1184}%
\special{pa 1483 1180}%
\special{pa 1515 1175}%
\special{pa 1547 1172}%
\special{pa 1578 1169}%
\special{pa 1610 1167}%
\special{pa 1642 1165}%
\special{pa 1674 1163}%
\special{pa 1706 1162}%
\special{pa 1738 1161}%
\special{pa 1770 1160}%
\special{pa 1802 1160}%
\special{pa 1834 1160}%
\special{pa 1866 1161}%
\special{pa 1898 1162}%
\special{pa 1930 1163}%
\special{pa 1962 1165}%
\special{pa 1994 1167}%
\special{pa 2026 1170}%
\special{pa 2058 1173}%
\special{pa 2089 1177}%
\special{pa 2121 1180}%
\special{pa 2153 1185}%
\special{pa 2185 1190}%
\special{pa 2216 1195}%
\special{pa 2247 1202}%
\special{pa 2279 1209}%
\special{pa 2310 1216}%
\special{pa 2341 1224}%
\special{pa 2371 1233}%
\special{pa 2402 1243}%
\special{pa 2432 1255}%
\special{pa 2461 1268}%
\special{pa 2489 1283}%
\special{pa 2503 1291}%
\special{sp}%
% VECTOR 2 0 3 0
% 2 246 167 251 169
%
\special{pn 8}%
\special{pa 2460 1270}%
\special{pa 2510 1290}%
\special{fp}%
\special{sh 1}%
\special{pa 2510 1290}%
\special{pa 2456 1247}%
\special{pa 2460 1270}%
\special{pa 2441 1284}%
\special{pa 2510 1290}%
\special{fp}%
% VECTOR 2 0 3 0
% 2 114 193 109 191
%
\special{pn 8}%
\special{pa 1140 1530}%
\special{pa 1090 1510}%
\special{fp}%
\special{sh 1}%
\special{pa 1090 1510}%
\special{pa 1144 1553}%
\special{pa 1140 1530}%
\special{pa 1159 1516}%
\special{pa 1090 1510}%
\special{fp}%
% ELLIPSE 2 0 3 0
% 4 298 182 259 213 264 195 262 166
%
\special{pn 8}%
\special{ar 2980 1420 390 310  3.6486912 6.2831853}%
\special{ar 2980 1420 390 310  0.0000000 2.7017501}%
% SPLINE2 2 0 3 0
% 42 945 1941 916 1974 882 2006 843 2032 799 2055 753 2073 703 2086 653		2093 601 2095 549 2092 498 2083 449 2068 402 2049 361 2025 322 1997 289		1964 262 1930 240 1891 226 1852 218 1811 216 1770 222 1728 234 1688 254		1651 279 1614 311 1581 347 1552 388 1527 433 1507 481 1491 531 1480 583		1476 635 1477 686 1482 737 1494 785 1511 828 1532 869 1558 906 1588 936		1622 961 1658 961 1658
%
\special{pn 8}%
\special{pa 945 1541}%
\special{pa 924 1565}%
\special{pa 902 1588}%
\special{pa 878 1609}%
\special{pa 851 1627}%
\special{pa 823 1643}%
\special{pa 795 1657}%
\special{pa 765 1669}%
\special{pa 734 1679}%
\special{pa 703 1686}%
\special{pa 672 1691}%
\special{pa 640 1694}%
\special{pa 608 1695}%
\special{pa 576 1694}%
\special{pa 544 1691}%
\special{pa 512 1686}%
\special{pa 481 1678}%
\special{pa 451 1669}%
\special{pa 421 1657}%
\special{pa 392 1644}%
\special{pa 364 1627}%
\special{pa 338 1609}%
\special{pa 313 1589}%
\special{pa 291 1566}%
\special{pa 270 1541}%
\special{pa 252 1515}%
\special{pa 238 1486}%
\special{pa 227 1456}%
\special{pa 220 1425}%
\special{pa 216 1393}%
\special{pa 217 1361}%
\special{pa 222 1330}%
\special{pa 230 1299}%
\special{pa 243 1270}%
\special{pa 260 1242}%
\special{pa 278 1216}%
\special{pa 299 1192}%
\special{pa 323 1171}%
\special{pa 348 1151}%
\special{pa 375 1134}%
\special{pa 404 1119}%
\special{pa 433 1107}%
\special{pa 464 1096}%
\special{pa 494 1087}%
\special{pa 526 1081}%
\special{pa 558 1077}%
\special{pa 590 1076}%
\special{pa 622 1076}%
\special{pa 653 1078}%
\special{pa 685 1082}%
\special{pa 717 1088}%
\special{pa 747 1097}%
\special{pa 778 1108}%
\special{pa 807 1121}%
\special{pa 835 1136}%
\special{pa 862 1153}%
\special{pa 888 1172}%
\special{pa 912 1194}%
\special{pa 933 1218}%
\special{pa 951 1244}%
\special{pa 961 1258}%
\special{sp}%
% VECTOR 2 0 3 0
% 2 93 162 98 166
%
\special{pn 8}%
\special{pa 930 1220}%
\special{pa 980 1260}%
\special{fp}%
\special{sh 1}%
\special{pa 980 1260}%
\special{pa 940 1203}%
\special{pa 938 1227}%
\special{pa 915 1234}%
\special{pa 980 1260}%
\special{fp}%
% STR 2 0 3 0
% 2 35 127 35 144
% $(0.6,0.7,0.8)$
\put(360,-990){$0.7$}%
% STR 2 0 3 0
% 2 146 136 146 153
% $(0.2,0.3,0.4)$
\put(1500,-1060){$0.3$}%
% STR 2 0 3 0
% 2 337 172 337 189
% $(0.5,0.6,0.7)$
\put(3400,-1460){$0.6$}%
% STR 2 0 3 0
% 2 145 206 145 223
% $(0.3,0.4,0.5)$
\put(1500,-1800){$0.4$}%
% STR 2 0 3 0
% 2 92 172 92 189
% $1$
\put(950,-1440){$1$}%
% STR 2 0 3 0
% 2 250 171 250 188
% $2$
\put(2540,-1440){$2$}%
% VECTOR 2 0 3 0
% 2 265 198 262 194
%
\special{pn 8}%
\special{pa 2650 1580}%
\special{pa 2620 1540}%
\special{fp}%
\special{sh 1}%
\special{pa 2620 1540}%
\special{pa 2644 1605}%
\special{pa 2652 1583}%
\special{pa 2676 1581}%
\special{pa 2620 1540}%
\special{fp}%
\end{picture}

\vspace*{4cm}
\end{figure}

\vspace*{-2.5cm}
\begin{flushright}
{\large 
$\displaystyle{
  \T{r}(f_1) = \begin{pmatrix}
               (\ \ 2, \ \ 3, \ 4) \\
               (-2, -1, \ 0)
               \end{pmatrix}
              \quad \quad 
}$
}
\end{flushright}

\vskip7mm \hskip4.5cm  {\bf (a)} Usual repair $f_1$


\bigskip


%\newpage
\vspace*{-3cm}

% ----------------------------------------------------------------------------
% \input fb.txt
\begin{figure}[htbp]
\unitlength 0.001in
%\vspace*{-2.0cm}
% \hspace*{2.5cm}
\begin{picture}(50,100)
% CIRCLE 2 0 3 0
% 4 100 180 114 182 115 180 115 180
%
\special{pn 8}%
\special{ar 1000 1400 141 141  0.0000000 6.2831853}%
% CIRCLE 2 0 3 0
% 4 260 180 274 182 275 180 275 180
%
\special{pn 8}%
\special{ar 2600 1400 141 141  0.0000000 6.2831853}%
% ELLIPSE 2 0 3 0
% 4 180 181 102 204 109 191 250 190
%
\special{pn 8}%
\special{ar 1800 1410 780 230  0.4168981 2.6949888}%
% SPLINE2 2 0 3 0
% 42 1087 1697 1106 1685 1127 1674 1151 1662 1177 1652 1204 1642 1234 1632	1265 1623 1299 1614 1333 1606 1370 1598 1407 1591 1446 1585 1487 1579 1528	1574 1570 1570 1613 1567 1656 1564 1700 1562 1744 1561 1788 1560 1833 1560	1877 1561 1921 1563 1965 1565 2008 1568 2050 1572 2092 1577 2133 1582 2172	1588 2211 1594 2248 1602 2283 1610 2318 1618 2350 1627 2381 1636 2409 1646	2436 1657 2461 1668 2483 1679 2503 1691 2503 1691
%
\special{pn 8}%
\special{pa 1087 1297}%
\special{pa 1114 1280}%
\special{pa 1143 1266}%
\special{pa 1172 1254}%
\special{pa 1202 1243}%
\special{pa 1233 1232}%
\special{pa 1263 1223}%
\special{pa 1294 1215}%
\special{pa 1326 1208}%
\special{pa 1357 1201}%
\special{pa 1388 1194}%
\special{pa 1420 1189}%
\special{pa 1451 1184}%
\special{pa 1483 1180}%
\special{pa 1515 1175}%
\special{pa 1547 1172}%
\special{pa 1578 1169}%
\special{pa 1610 1167}%
\special{pa 1642 1165}%
\special{pa 1674 1163}%
\special{pa 1706 1162}%
\special{pa 1738 1161}%
\special{pa 1770 1160}%
\special{pa 1802 1160}%
\special{pa 1834 1160}%
\special{pa 1866 1161}%
\special{pa 1898 1162}%
\special{pa 1930 1163}%
\special{pa 1962 1165}%
\special{pa 1994 1167}%
\special{pa 2026 1170}%
\special{pa 2058 1173}%
\special{pa 2089 1177}%
\special{pa 2121 1180}%
\special{pa 2153 1185}%
\special{pa 2185 1190}%
\special{pa 2216 1195}%
\special{pa 2247 1202}%
\special{pa 2279 1209}%
\special{pa 2310 1216}%
\special{pa 2341 1224}%
\special{pa 2371 1233}%
\special{pa 2402 1243}%
\special{pa 2432 1255}%
\special{pa 2461 1268}%
\special{pa 2489 1283}%
\special{pa 2503 1291}%
\special{sp}%
% VECTOR 2 0 3 0
% 2 246 167 251 169
%
\special{pn 8}%
\special{pa 2460 1270}%
\special{pa 2510 1290}%
\special{fp}%
\special{sh 1}%
\special{pa 2510 1290}%
\special{pa 2456 1247}%
\special{pa 2460 1270}%
\special{pa 2441 1284}%
\special{pa 2510 1290}%
\special{fp}%
% VECTOR 2 0 3 0
% 2 114 193 109 191
%
\special{pn 8}%
\special{pa 1140 1530}%
\special{pa 1090 1510}%
\special{fp}%
\special{sh 1}%
\special{pa 1090 1510}%
\special{pa 1144 1553}%
\special{pa 1140 1530}%
\special{pa 1159 1516}%
\special{pa 1090 1510}%
\special{fp}%
% ELLIPSE 2 0 3 0
% 4 298 182 259 213 264 195 262 166
%
\special{pn 8}%
\special{ar 2980 1420 390 310  3.6486912 6.2831853}%
\special{ar 2980 1420 390 310  0.0000000 2.7017501}%
% SPLINE2 2 0 3 0
% 42 945 1941 916 1974 882 2006 843 2032 799 2055 753 2073 703 2086 653		2093 601 2095 549 2092 498 2083 449 2068 402 2049 361 2025 322 1997 289		1964 262 1930 240 1891 226 1852 218 1811 216 1770 222 1728 234 1688 254		1651 279 1614 311 1581 347 1552 388 1527 433 1507 481 1491 531 1480 583		1476 635 1477 686 1482 737 1494 785 1511 828 1532 869 1558 906 1588 936		1622 961 1658 961 1658
%
\special{pn 8}%
\special{pa 945 1541}%
\special{pa 924 1565}%
\special{pa 902 1588}%
\special{pa 878 1609}%
\special{pa 851 1627}%
\special{pa 823 1643}%
\special{pa 795 1657}%
\special{pa 765 1669}%
\special{pa 734 1679}%
\special{pa 703 1686}%
\special{pa 672 1691}%
\special{pa 640 1694}%
\special{pa 608 1695}%
\special{pa 576 1694}%
\special{pa 544 1691}%
\special{pa 512 1686}%
\special{pa 481 1678}%
\special{pa 451 1669}%
\special{pa 421 1657}%
\special{pa 392 1644}%
\special{pa 364 1627}%
\special{pa 338 1609}%
\special{pa 313 1589}%
\special{pa 291 1566}%
\special{pa 270 1541}%
\special{pa 252 1515}%
\special{pa 238 1486}%
\special{pa 227 1456}%
\special{pa 220 1425}%
\special{pa 216 1393}%
\special{pa 217 1361}%
\special{pa 222 1330}%
\special{pa 230 1299}%
\special{pa 243 1270}%
\special{pa 260 1242}%
\special{pa 278 1216}%
\special{pa 299 1192}%
\special{pa 323 1171}%
\special{pa 348 1151}%
\special{pa 375 1134}%
\special{pa 404 1119}%
\special{pa 433 1107}%
\special{pa 464 1096}%
\special{pa 494 1087}%
\special{pa 526 1081}%
\special{pa 558 1077}%
\special{pa 590 1076}%
\special{pa 622 1076}%
\special{pa 653 1078}%
\special{pa 685 1082}%
\special{pa 717 1088}%
\special{pa 747 1097}%
\special{pa 778 1108}%
\special{pa 807 1121}%
\special{pa 835 1136}%
\special{pa 862 1153}%
\special{pa 888 1172}%
\special{pa 912 1194}%
\special{pa 933 1218}%
\special{pa 951 1244}%
\special{pa 961 1258}%
\special{sp}%
% VECTOR 2 0 3 0
% 2 93 162 98 166
%
\special{pn 8}%
\special{pa 930 1220}%
\special{pa 980 1260}%
\special{fp}%
\special{sh 1}%
\special{pa 980 1260}%
\special{pa 940 1203}%
\special{pa 938 1227}%
\special{pa 915 1234}%
\special{pa 980 1260}%
\special{fp}%
% STR 2 0 3 0
% 2 35 127 35 144
% $(0.6,0.7,0.8)$
\put(360,-990){$0.7$}%
% STR 2 0 3 0
% 2 146 136 146 153
% $(0.2,0.3,0.4)$
\put(1500,-1060){$0.3$}%
% STR 2 0 3 0
% 2 337 172 337 189
% $(0.3,0.4,0.5)$
\put(3400,-1460){$0.4$}%
% STR 2 0 3 0
% 2 145 206 145 223
% $(0.5,0.6,0.7)$
\put(1500,-1800){$0.6$}%
% STR 2 0 3 0
% 2 92 172 92 189
% $1$
\put(950,-1440){$1$}%
% STR 2 0 3 0
% 2 250 171 250 188
% $2$
\put(2540,-1440){$2$}%
% VECTOR 2 0 3 0
% 2 265 198 262 194
%
\special{pn 8}%
\special{pa 2650 1580}%
\special{pa 2620 1540}%
\special{fp}%
\special{sh 1}%
\special{pa 2620 1540}%
\special{pa 2644 1605}%
\special{pa 2652 1583}%
\special{pa 2676 1581}%
\special{pa 2620 1540}%
\special{fp}%
\end{picture}

\vspace*{4cm}

\end{figure}

\vspace*{-2.5cm}
\begin{flushright}
{\large 
$\displaystyle{
  \T{r}(f_2) = \begin{pmatrix}
               (\ \ 2, \ \ 3, \ \ 4) \\
               (-3, -2, -1)
               \end{pmatrix}
              \quad \quad
}$
}
\end{flushright}




\vspace*{2.2em}
\hskip5.5cm 
{\bf (b)} Rapid repair $f_2$

\vskip0.2cm \hskip3.0cm 
{\bf Figure.1} \;Transition diagrams and fuzzy rewards.

% Fig.1 ---------------------------------------------------------------------

\bigskip
\bigskip

Applying Theorem 4.1, we obtain the infinite horizon FEDR 
as a unique solution of (4.3). 
So, putting 
$$
\T{\phi}(f_1)_\af = 
([x^1_{\af}, y^1_{\af}], [x^2_{\af}, y^2_{\af}])',
$$
the $\af$-cut interval equations (4.4) with $\beta = 0.9$ become: 
$$
\left\{ 
\begin{array}{l} 

\vspace*{0.3em}
\displaystyle{
x^1_{\af} =  2 + \af + 0.9 (0.7x^1_{\af} + 0.3x^2_{\af})}   \\ 

\vspace*{0.3em}
\displaystyle{
y^1_{\af} =  4 - \af + 0.9 (0.7y^1_{\af} + 0.3y^2_{\af})}  \\ 

\vspace*{0.3em}
\displaystyle{
x^2_{\af} = -2 + \af + 0.9 (0.4x^1_{\af} + 0.6x^2_{\af})}  \\ 

\vspace*{0.3em}
\displaystyle{
y^2_{\af} =    - \af + 0.9 (0.4y^1_{\af} + 0.6y^2_{\af})}
\end{array}
\right.
$$
After a simple calculation, we obtain 
$$
\T{\phi}(f_1)_\af =\Big(
  [10\af + \frac{380}{73}, \frac{1840}{73} - 10\af], 
  [10\af - \frac{20}{73}, \frac{1440}{73} - 10\af] 
                    \Big)', 
$$
which leads to 

\vspace*{-0.5em}
$$
\T{\phi}(f_1) =\Big(
 (\frac{380}{73}, \frac{1110}{73}, \frac{1840}{73}), 
 (-\frac{20}{73}, \frac{710}{73},  \frac{1440}{73}) 
                   \Big)'. 
$$

% ----------------------------------------------------------------------------
\section*{5. Pareto optimality} 
Here, we confine our attention to the class of stationary policies, 
which simplifies our discussion in the sequel. 
Let $K$ be a convex cone in $\BR^p$. 
A policy $f^* \in \Pi_F$ is called Pareto optimal 
if there does not exist $f \in \Pi_F$ such that 
$\T{\phi}(f^*) \ \inc_K  \ \T{\phi}(f)$.
In this section, 
we derive the optimal equation, 
by which  Pareto optimal policies are characterized. 

The following important result is crucial to the development 
in the characterization of Pareto optimality. 

\bigskip
\noindent
{\bf Lemma 5.1.} \ 
{\sl 
For any $f, g \in F$, suppose that 

\vspace*{-1.5em}
$$
\T{\phi}(f) 
\quad 
\Big\{ {\inc_K \atop \prec_K} \Big\} 
\quad 
U_g \T{\phi}(f). 
\leqno{(5.1)}
$$

\vspace*{-1em}
Then, it holds that 

\vspace*{-1.5em}
$$
\T{\phi}(f) 
\quad 
\Big\{ {\inc_K \atop \prec_K} \Big\} 
\quad 
\T{\phi}(g). 
\leqno{(5.2)}
$$
}

\vspace*{-0.5em}
\noindent
\Pf
Suppose that 
$\T{\phi}(f) \ \{ {\inc_K \atop \prec_K} \} \ U_g \T{\phi}(f)$.  
Then, we have from Lemma 4.3 that 
$$
\T{\phi}(f) \quad 
\Big\{ {\inc_K \atop \prec_K} \Big\} \quad 
U_g \T{\phi}(f) \quad 
\inc \quad 
U_g^l \T{\psi}(f) \quad 
(l \ge 2),
$$
So, taking the limit in the above as $l \to \infty$, 
(5.2) follows from Corollary 4.1 and Lemma 2.4. 
\qed

\bigskip

Let $D$ be an arbitrary subset of $\C{F}(\BR^p)^n$. 
A point $\T{u} \in D$ is called an efficient element of $D$ 
with respect to $\inc_K $ on $\C{F}(\BR^p)^n$ if and only if 
it holds that there does not exist $\T{v} \in D$ such that 
$\T{u} \prec_K \T{v}$.
We denote by ${\rm eff}(D)$ 
the set of all elements of $D$ efficient with respect to 
$\inc_K $ on $\C{F}(\BR^p)^n$.
For any $\T{u} \in \C{F}(\BR^p)^n$, let
$
\C{U}(\T{u}) := \mbox{eff} ( \{ U_f \T{u} \mid f \in F \}).
$
Note that $\C{U}(\T{u}) \subset \C{F}(\BR^p)^n$. 

Here, we consider the following fuzzy equation 
including efficient fuzzy functions 
$\C{U}(\cdot)$ on $\C{F}(\BR^p)^n$:
$$
\T{u} \in \C{U}(\T{u}), \quad 
\T{u} \in \C{F}(\BR^p)^n.
\leqno{(5.3)}
$$
The equation (5.3) is called an optimality equation, 
by which Pareto optimal policies are characterized. 
A solution $\T{u}$ of (5.3) is called maximal 
if there does not exist any solution $\T{u}'$ of (5.3) 
such that $\T{u} \prec_K \T{u}'$. 
Pareto optimal policies are characterized by maximal solutions 
of the optimality equation (5.3). 

\bigskip
\noindent
{\bf Theorem 5.1.}\ 
{\sl 
A policy $f$ is Pareto optimal if and only if 
a fixed point of the corresponding $U_f$, $\T{\phi}(f)$, 
is a maximal solution to the optimal equation (5.3). 
}

\medskip
\Pf
The proof of \lq\lq only if \rq\rq part is easily obtained from Lemma 5.1. 
In order to prove \lq\lq if \rq\rq part, 
suppose that $\T{\phi}(f)$ is a maximal solution of (5.3) 
but $f$ is not Pareto optimal. 
Then, there exists 
$f^{(1)} \in F$ such that $\T{\phi}(f) \prec_K \T{\phi}(f^{(1)})$. 

Now, suppose that 
$\T{\phi}(f^{(1)}) \not\in \mbox{eff}(\T{\phi}(f^{(1)}))$. 
This assumption assures that there exists 
$f^{(2)} \in F$ 
satisfying $\T{\phi}(f^{(1)}) \prec_K U_{f^{(2)}}\T{\phi}(f^{(1)})$, 
which implies from (5.1) that 
$\T{\phi}(f^{(1)}) \prec_K \T{\phi}(f^{(2)})$. 
By repeating this method successively, 
we come to the conclusion that there exists 
$f^{(l)} \in F$ 
such that 
$\T{\phi}(f) \prec_K \T{\phi}(f^{(l)})$ and 
$\T{\phi}(f^{(l)})$ satisfies (5.3), 
which contradicts that $\T{\phi}(f)$ is maximal, as required.
\qed

\bigskip
\noindent
{\bf Remark.} \ 
For vector-valued discounted MDPs, 
Furukawa\cite{Fur80} and White\cite{Whi82} 
had derived the optimality equation 
including efficient set-function on $\BR^p$, 
by that Pareto optimal policies are characterized. 
The form of the optimal equation (5.3) is corresponding 
to a fuzzy version of MDPs. 

\bigskip
For the machine maintenance problem given in Section 4, 
we find that 
$$
U_{f_2}\T{\phi}(f_1) = \Big( 
 (\frac{380}{73}, \frac{1110}{73}, \frac{1840}{73}), 
 (-\frac{21}{73}, \frac{709}{73},  \frac{1439}{73}) 
                   \Big)', 
$$

\vspace*{-0.5em}
Recall that 

\vspace*{-0.5em}
$$
U_{f_1}\T{\phi}(f_1) = \T{\phi}(f_1) =\Big(
 (\frac{380}{73}, \frac{1110}{73}, \frac{1840}{73}), 
 (-\frac{20}{73}, \frac{710}{73},  \frac{1440}{73}) 
                   \Big)', 
$$
which satisfies 
$U_{f_2}\T{\phi}(f_1) \prec_1 \T{\phi}(f_1)$,
where $\prec_1$ is the fuzzy max order on $\C{F}(\BR)^2$ 
and corresponding to $\inc_K$ in case of $K = [0,\infty)$.

Thus, 
$\T{\phi}(f_1) \in {\rm eff}( \{ U_f\T{\phi}(f_1) \mid f \in F)$,
so that from Theorem 5.1 $f_1$ is Pareto optimal. 
In fact, we can find, by solving (4.3) or (4.4) for $f_2$, that 
$$
\T{\phi}(f_2) =\Big(
 (\frac{470}{91}, \frac{1380}{91}, \frac{2290}{91}), 
 (-\frac{30}{91}, \frac{880}{91},  \frac{1790}{91}) 
                   \Big)', 
\mbox{\ and \ }
\T{\phi}(f_2) \prec_1 \T{\phi}(f_1).
$$

%\newpage

%------------------------- References  ---------------------------------------
\begin{thebibliography}{99}
%\baselineskip 1mm

%1
\bibitem{Aum65} Aumann, R. J.,
Integrals of set-valued functions,
{\it J. Math. Anal. Appl.} {\bf 12} (1965), 1-12.

%2
\bibitem{Bla62} Blackwell, D.,
Discrete dynamic programming,
{\it Ann. Math. Stat.} {\bf 33} (1962), 719-726.

%3
\bibitem{DK94} Diamond, P. and Kloeden, P.,
{\it Metric Spaces of Fuzzy Sets, Theory and Applications},
(1994), World Scientific.

%4
\bibitem{Fur80} Furukawa, N.,
Characterization of optimal policies 
in vector-valued Markovian decision processes,
{\it Math. Oper. Res.} {\bf 5} (1980), 271-279.

%5
\bibitem{Fur97} Furukawa, N.,
Parametric orders on fuzzy numbers and their roles 
in fuzzy optimization problems,
{\it Optimization} {\bf 40} (1997), 171-192.

%6
\bibitem{How60} Howard, R.,
{\it Dynamic Programming and Markov processes},
(1960), MIT Press, Cambridge MA.

%7
\bibitem{KYNY96} Kurano, M., Yasuda, M., Nakagami, J. and Yoshida, Y.,
Markov-type fuzzy decision processes with a discounted reward 
on a closed interval,
{\it European J. Oper. Res.}, {\bf 92} (1996), 649-662.

%8
\bibitem{KSHH98} Kurano, M., Song, J., Hosaka, M. and Huang, Y.,
Controlled Markov Set-Chains with Discounting,
{\it J. Appl. Prob.}, {\bf 35} (1998), 293-302.

%9
\bibitem{KYN99} Kurano, M., Yasuda, M. and Nakagami, J.,
Interval methods for uncertain Markov decision processes, 
{\it submitted to the International Workshop on Markov Processes and Controlled Markov Chains}, Changsha, Husan, China on August 22-28, 1999.

%10
\bibitem{KYNY00a} 
Kurano, M., Yasuda, M., Nakagami, J. and Yoshida, Y.,
Ordering of fuzzy sets -- A brief survey and new results, 
{\it J. Operations Research Society of Japan}, 
{\bf 43} (2000), 138-148.

%11
\bibitem{KYNY00b} 
Kurano, M., Yasuda, M., Nakagami, J. and Yoshida, Y.,
A fuzzy treatment of uncertain Markov decision processes, 
Proceedings of ASSM2000 
International Conference on Applied Stochastic System Modeling
(Kyoto, March 2000), 148-157.

%12
\bibitem{MinOsa70} Mine, H. and Osaki, S.,
{\it Markov Decision Processes},
(1970), Elsevier, Amsterdam. 

%13
\bibitem{Nov89} Nov{\'a}k, V.,
{\it Fuzzy sets and their applications},
(1989), Adam Hilger, Bristole-Boston. 

%14
\bibitem{PR86} Puri, M. L. and Ralesca, D. A., 
Fuzzy random variable, 
{\it J. Math. Anal. Appl.}, {\bf 114} (1986), 402-422.

%15
\bibitem{Put94} Puterman, M. L.,
{\it Markov decision processes: Discrete Stochastic Dynamic Programming},
(1994), John Wiley \& Sons, INC.

%16
\bibitem{RR85}
J.Ram\'{\i}k and J.\u{R}im\'{a}nek,
Inequality relation between fuzzy numbers and its use  in fuzzy optimization,
{\it Fuzzy Sets and Systems},  
{\bf 16} (1985), 123-138.

%17
\bibitem{Sto98} Stow{\'\i}nski, R., (ed.),
{\it Fuzzy Sets in Decision Analysis, Operation Research and Statistics},
(1998), Kluwer Academic Publishers.

%18
\bibitem{Whi82} White, D. J.,
Multi-objective infinite-horizon discounted Markov Decision Processes,
{\it J. Math. Anal. Appl.},  {\bf 89} (1982), 639-647.

%19
\bibitem{Yos98} Yoshida, Y.,
A time-average fuzzy reward criterion in fuzzy decision processes,
{\it Information Sciences}, {\bf 110} (1998), 103-112.

%20
\bibitem{Zad65} Zadeh, L. A.,
Fuzzy sets,
{\it Inform. and Control}, {\bf 8} (1965), 338-353.

\end{thebibliography}

\end{document} %---------------------------------------------------------------


