\documentclass[11pt,twoside]{article}\makeatletter

\IfFileExists{xcolor.sty}%
  {\RequirePackage{xcolor}}%
  {\RequirePackage{color}}
\usepackage{colortbl}
\usepackage{wrapfig}
\usepackage{ifxetex}
\ifxetex
  \usepackage{fontspec}
  \usepackage{xunicode}
  \catcode`⃥=\active \def⃥{\textbackslash}
  \catcode`❴=\active \def❴{\{}
  \catcode`❵=\active \def❵{\}}
  \def\textJapanese{\fontspec{Noto Sans CJK JP}}
  \def\textChinese{\fontspec{Noto Sans CJK SC}}
  \def\textKorean{\fontspec{Noto Sans CJK KR}}
  \setmonofont{DejaVu Sans Mono}
  
\else
  \IfFileExists{utf8x.def}%
   {\usepackage[utf8x]{inputenc}
      \PrerenderUnicode{–}
    }%
   {\usepackage[utf8]{inputenc}}
  \usepackage[english]{babel}
  \usepackage[T1]{fontenc}
  \usepackage{float}
  \usepackage[]{ucs}
  \uc@dclc{8421}{default}{\textbackslash }
  \uc@dclc{10100}{default}{\{}
  \uc@dclc{10101}{default}{\}}
  \uc@dclc{8491}{default}{\AA{}}
  \uc@dclc{8239}{default}{\,}
  \uc@dclc{20154}{default}{ }
  \uc@dclc{10148}{default}{>}
  \def\textschwa{\rotatebox{-90}{e}}
  \def\textJapanese{}
  \def\textChinese{}
  \IfFileExists{tipa.sty}{\usepackage{tipa}}{}
\fi
\def\exampleFont{\ttfamily\small}
\DeclareTextSymbol{\textpi}{OML}{25}
\usepackage{relsize}
\RequirePackage{array}
\def\@testpach{\@chclass
 \ifnum \@lastchclass=6 \@ne \@chnum \@ne \else
  \ifnum \@lastchclass=7 5 \else
   \ifnum \@lastchclass=8 \tw@ \else
    \ifnum \@lastchclass=9 \thr@@
   \else \z@
   \ifnum \@lastchclass = 10 \else
   \edef\@nextchar{\expandafter\string\@nextchar}%
   \@chnum
   \if \@nextchar c\z@ \else
    \if \@nextchar l\@ne \else
     \if \@nextchar r\tw@ \else
   \z@ \@chclass
   \if\@nextchar |\@ne \else
    \if \@nextchar !6 \else
     \if \@nextchar @7 \else
      \if \@nextchar (8 \else
       \if \@nextchar )9 \else
  10
  \@chnum
  \if \@nextchar m\thr@@\else
   \if \@nextchar p4 \else
    \if \@nextchar b5 \else
   \z@ \@chclass \z@ \@preamerr \z@ \fi \fi \fi \fi
   \fi \fi  \fi  \fi  \fi  \fi  \fi \fi \fi \fi \fi \fi}
\gdef\arraybackslash{\let\\=\@arraycr}
\def\@textsubscript#1{{\m@th\ensuremath{_{\mbox{\fontsize\sf@size\z@#1}}}}}
\def\Panel#1#2#3#4{\multicolumn{#3}{){\columncolor{#2}}#4}{#1}}
\def\abbr{}
\def\corr{}
\def\expan{}
\def\gap{}
\def\orig{}
\def\reg{}
\def\ref{}
\def\sic{}
\def\persName{}\def\name{}
\def\placeName{}
\def\orgName{}
\def\textcal#1{{\fontspec{Lucida Calligraphy}#1}}
\def\textgothic#1{{\fontspec{Lucida Blackletter}#1}}
\def\textlarge#1{{\large #1}}
\def\textoverbar#1{\ensuremath{\overline{#1}}}
\def\textquoted#1{‘#1’}
\def\textsmall#1{{\small #1}}
\def\textsubscript#1{\@textsubscript{\selectfont#1}}
\def\textxi{\ensuremath{\xi}}
\def\titlem{\itshape}
\newenvironment{biblfree}{}{\ifvmode\par\fi }
\newenvironment{bibl}{}{}
\newenvironment{byline}{\vskip6pt\itshape\fontsize{16pt}{18pt}\selectfont}{\par }
\newenvironment{citbibl}{}{\ifvmode\par\fi }
\newenvironment{docAuthor}{\ifvmode\vskip4pt\fontsize{16pt}{18pt}\selectfont\fi\itshape}{\ifvmode\par\fi }
\newenvironment{docDate}{}{\ifvmode\par\fi }
\newenvironment{docImprint}{\vskip 6pt}{\ifvmode\par\fi }
\newenvironment{docTitle}{\vskip6pt\bfseries\fontsize{22pt}{25pt}\selectfont}{\par }
\newenvironment{msHead}{\vskip 6pt}{\par}
\newenvironment{msItem}{\vskip 6pt}{\par}
\newenvironment{rubric}{}{}
\newenvironment{titlePart}{}{\par }

\newcolumntype{L}[1]{){\raggedright\arraybackslash}p{#1}}
\newcolumntype{C}[1]{){\centering\arraybackslash}p{#1}}
\newcolumntype{R}[1]{){\raggedleft\arraybackslash}p{#1}}
\newcolumntype{P}[1]{){\arraybackslash}p{#1}}
\newcolumntype{B}[1]{){\arraybackslash}b{#1}}
\newcolumntype{M}[1]{){\arraybackslash}m{#1}}
\definecolor{label}{gray}{0.75}
\def\unusedattribute#1{\sout{\textcolor{label}{#1}}}
\DeclareRobustCommand*{\xref}{\hyper@normalise\xref@}
\def\xref@#1#2{\hyper@linkurl{#2}{#1}}
\begingroup
\catcode`\_=\active
\gdef_#1{\ensuremath{\sb{\mathrm{#1}}}}
\endgroup
\mathcode`\_=\string"8000
\catcode`\_=12\relax

\usepackage[a4paper,twoside,lmargin=1in,rmargin=1in,tmargin=1in,bmargin=1in,marginparwidth=0.75in]{geometry}
\usepackage{framed}

\definecolor{shadecolor}{gray}{0.95}
\usepackage{longtable}
\usepackage[normalem]{ulem}
\usepackage{fancyvrb}
\usepackage{fancyhdr}
\usepackage{graphicx}
\usepackage{marginnote}

\renewcommand{\@cite}[1]{#1}


\renewcommand*{\marginfont}{\itshape\footnotesize}

\def\Gin@extensions{.pdf,.png,.jpg,.mps,.tif}

  \pagestyle{fancy}

\usepackage[pdftitle={L-lysine Neuro-Dynamic Optimal Control},
 pdfauthor={}]{hyperref}
\hyperbaseurl{}

	 \paperwidth210mm
	 \paperheight297mm
              
\def\@pnumwidth{1.55em}
\def\@tocrmarg {2.55em}
\def\@dotsep{4.5}
\setcounter{tocdepth}{3}
\clubpenalty=8000
\emergencystretch 3em
\hbadness=4000
\hyphenpenalty=400
\pretolerance=750
\tolerance=2000
\vbadness=4000
\widowpenalty=10000

\renewcommand\section{\@startsection {section}{1}{\z@}%
     {-1.75ex \@plus -0.5ex \@minus -.2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\Large\bfseries}}
\renewcommand\subsection{\@startsection{subsection}{2}{\z@}%
     {-1.75ex\@plus -0.5ex \@minus- .2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\Large}}
\renewcommand\subsubsection{\@startsection{subsubsection}{3}{\z@}%
     {-1.5ex\@plus -0.35ex \@minus -.2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\large}}
\renewcommand\paragraph{\@startsection{paragraph}{4}{\z@}%
     {-1ex \@plus-0.35ex \@minus -0.2ex}%
     {0.5ex \@plus .2ex}%
     {\reset@font\normalsize}}
\renewcommand\subparagraph{\@startsection{subparagraph}{5}{\parindent}%
     {1.5ex \@plus1ex \@minus .2ex}%
     {-1em}%
     {\reset@font\normalsize\bfseries}}


\def\l@section#1#2{\addpenalty{\@secpenalty} \addvspace{1.0em plus 1pt}
 \@tempdima 1.5em \begingroup
 \parindent \z@ \rightskip \@pnumwidth 
 \parfillskip -\@pnumwidth 
 \bfseries \leavevmode #1\hfil \hbox to\@pnumwidth{\hss #2}\par
 \endgroup}
\def\l@subsection{\@dottedtocline{2}{1.5em}{2.3em}}
\def\l@subsubsection{\@dottedtocline{3}{3.8em}{3.2em}}
\def\l@paragraph{\@dottedtocline{4}{7.0em}{4.1em}}
\def\l@subparagraph{\@dottedtocline{5}{10em}{5em}}
\@ifundefined{c@section}{\newcounter{section}}{}
\@ifundefined{c@chapter}{\newcounter{chapter}}{}
\newif\if@mainmatter 
\@mainmattertrue
\def\chaptername{Chapter}
\def\frontmatter{%
  \pagenumbering{roman}
  \def\thechapter{\@roman\c@chapter}
  \def\theHchapter{\roman{chapter}}
  \def\thesection{\@roman\c@section}
  \def\theHsection{\roman{section}}
  \def\@chapapp{}%
}
\def\mainmatter{%
  \cleardoublepage
  \def\thechapter{\@arabic\c@chapter}
  \setcounter{chapter}{0}
  \setcounter{section}{0}
  \pagenumbering{arabic}
  \setcounter{secnumdepth}{6}
  \def\@chapapp{\chaptername}%
  \def\theHchapter{\arabic{chapter}}
  \def\thesection{\@arabic\c@section}
  \def\theHsection{\arabic{section}}
}
\def\backmatter{%
  \cleardoublepage
  \setcounter{chapter}{0}
  \setcounter{section}{0}
  \setcounter{secnumdepth}{2}
  \def\@chapapp{\appendixname}%
  \def\thechapter{\@Alph\c@chapter}
  \def\theHchapter{\Alph{chapter}}
  \appendix
}
\newenvironment{bibitemlist}[1]{%
   \list{\@biblabel{\@arabic\c@enumiv}}%
       {\settowidth\labelwidth{\@biblabel{#1}}%
        \leftmargin\labelwidth
        \advance\leftmargin\labelsep
        \@openbib@code
        \usecounter{enumiv}%
        \let\p@enumiv\@empty
        \renewcommand\theenumiv{\@arabic\c@enumiv}%
	}%
  \sloppy
  \clubpenalty4000
  \@clubpenalty \clubpenalty
  \widowpenalty4000%
  \sfcode`\.\@m}%
  {\def\@noitemerr
    {\@latex@warning{Empty `bibitemlist' environment}}%
    \endlist}

\def\tableofcontents{\section*{\contentsname}\@starttoc{toc}}
\parskip0pt
\parindent1em
\def\Panel#1#2#3#4{\multicolumn{#3}{){\columncolor{#2}}#4}{#1}}
\newenvironment{reflist}{%
  \begin{raggedright}\begin{list}{}
  {%
   \setlength{\topsep}{0pt}%
   \setlength{\rightmargin}{0.25in}%
   \setlength{\itemsep}{0pt}%
   \setlength{\itemindent}{0pt}%
   \setlength{\parskip}{0pt}%
   \setlength{\parsep}{2pt}%
   \def\makelabel##1{\itshape ##1}}%
  }
  {\end{list}\end{raggedright}}
\newenvironment{sansreflist}{%
  \begin{raggedright}\begin{list}{}
  {%
   \setlength{\topsep}{0pt}%
   \setlength{\rightmargin}{0.25in}%
   \setlength{\itemindent}{0pt}%
   \setlength{\parskip}{0pt}%
   \setlength{\itemsep}{0pt}%
   \setlength{\parsep}{2pt}%
   \def\makelabel##1{\upshape ##1}}%
  }
  {\end{list}\end{raggedright}}
\newenvironment{specHead}[2]%
 {\vspace{20pt}\hrule\vspace{10pt}%
  \phantomsection\label{#1}\markright{#2}%

  \pdfbookmark[2]{#2}{#1}%
  \hspace{-0.75in}{\bfseries\fontsize{16pt}{18pt}\selectfont#2}%
  }{}
      \def\TheFullDate{2011-12-04 (revised: 04 December 2011)}
\def\TheID{\makeatother }
\def\TheDate{2011-12-04}
\title{L-lysine Neuro-Dynamic Optimal Control}
\author{}\makeatletter 
\makeatletter
\newcommand*{\cleartoleftpage}{%
  \clearpage
    \if@twoside
    \ifodd\c@page
      \hbox{}\newpage
      \if@twocolumn
        \hbox{}\newpage
      \fi
    \fi
  \fi
}
\makeatother
\makeatletter
\thispagestyle{empty}
\markright{\@title}\markboth{\@title}{\@author}
\renewcommand\small{\@setfontsize\small{9pt}{11pt}\abovedisplayskip 8.5\p@ plus3\p@ minus4\p@
\belowdisplayskip \abovedisplayskip
\abovedisplayshortskip \z@ plus2\p@
\belowdisplayshortskip 4\p@ plus2\p@ minus2\p@
\def\@listi{\leftmargin\leftmargini
               \topsep 2\p@ plus1\p@ minus1\p@
               \parsep 2\p@ plus\p@ minus\p@
               \itemsep 1pt}
}
\makeatother
\fvset{frame=single,numberblanklines=false,xleftmargin=5mm,xrightmargin=5mm}
\fancyhf{} 
\setlength{\headheight}{14pt}
\fancyhead[LE]{\bfseries\leftmark} 
\fancyhead[RO]{\bfseries\rightmark} 
\fancyfoot[RO]{}
\fancyfoot[CO]{\thepage}
\fancyfoot[LO]{\TheID}
\fancyfoot[LE]{}
\fancyfoot[CE]{\thepage}
\fancyfoot[RE]{\TheID}
\hypersetup{citebordercolor=0.75 0.75 0.75,linkbordercolor=0.75 0.75 0.75,urlbordercolor=0.75 0.75 0.75,bookmarksnumbered=true}
\fancypagestyle{plain}{\fancyhead{}\renewcommand{\headrulewidth}{0pt}}

\date{}
\usepackage{authblk}

\providecommand{\keywords}[1]
{
\footnotesize
  \textbf{\textit{Index terms---}} #1
}

\usepackage{graphicx,xcolor}
\definecolor{GJBlue}{HTML}{273B81}
\definecolor{GJLightBlue}{HTML}{0A9DD9}
\definecolor{GJMediumGrey}{HTML}{6D6E70}
\definecolor{GJLightGrey}{HTML}{929497} 

\renewenvironment{abstract}{%
   \setlength{\parindent}{0pt}\raggedright
   \textcolor{GJMediumGrey}{\rule{\textwidth}{2pt}}
   \vskip16pt
   \textcolor{GJBlue}{\large\bfseries\abstractname\space}
}{%   
   \vskip8pt
   \textcolor{GJMediumGrey}{\rule{\textwidth}{2pt}}
   \vskip16pt
}

\usepackage[absolute,overlay]{textpos}

\makeatother 
      \usepackage{lineno}
      \linenumbers
      
\begin{document}

             \author[1]{Dr. Tatiana  Ilkova}

             \author[2]{Dr. Mitko  Petrov}

             \affil[1]{  Institute of Biophysics and Biomedical Engineering, Bulgarian Academy of Sciences}

\renewcommand\Authands{ and }

\date{\small \em Received: 27 October 2011 Accepted: 20 November 2011 Published: 4 December 2011}

\maketitle


\begin{abstract}
        


In this paper Neuro-dynamic programming (NDP) is proposed as an alternative to alleviate the ?curse of dimensionality? of the Dynamic programming (DP) for optimal control of a fed-batch fermentation process in the L-lysine production. The most effective and cheapest method for the Llysine biosynthesis (in biological active form) is the microbiological method via a direct fermentation. In this paper an optimization method of the L-lysine production from strain Brevibacterium flavum 22LD is used and that is NDP. The results show that the quality of L-lysine enhances at the end of the process. The proposed method is particularly simple to implement and can be applied for on-line optimization.

\end{abstract}


\keywords{dynamic programming, neural network, Llysine fermentation, optimal control}

\begin{textblock*}{18cm}(1cm,1cm) % {block width} (coords) 
\textcolor{GJBlue}{\LARGE Global Journals \LaTeX\ JournalKaleidoscope\texttrademark}
\end{textblock*}

\begin{textblock*}{18cm}(1.4cm,1.5cm) % {block width} (coords) 
\textcolor{GJBlue}{\footnotesize \\ Artificial Intelligence formulated this projection for compatibility purposes from the original article published at Global Journals. However, this technology is currently in beta. \emph{Therefore, kindly ignore odd layouts, missed formulae, text, tables, or figures.}}
\end{textblock*}


\let\tabcellsep& 	 	 		 
\section[{INTRODUCTION}]{INTRODUCTION}\par
-lysine is an essential amino acid, which means that it is essential to human health, but cannot be produced by the body. For this reason L-lysine must be obtained from food. Amino acids are the building blocks of the protein. Lysine is important for proper growth and it plays an essential role in the production of carnitine, which is a nutrient responsible for converting fatty acids into energy and helping to lower cholesterol.\par
The insufficient L-lysine quantity in the fodders reduces the biological value of the fodder doses, it also reduces the weight increase and the further productiveness of the agricultural animals, decreases the fodder quality, used for a kilogram growth and decreases the product quantity from animal origin. Llysine is also used in the food industry for farming, in the medicine as a component of the infusion solution (blood substitutes) and as generally strengthening patent medicines. Lysine appears to help the body absorb and conserve calcium and it plays an important role in the formation of collagen, a substance which is important for the bones and connective tissues including skin, tendon, and cartilage \hyperref[b6]{(Anastassiadis, 2007)}.\par
Amino acids are the basic bioelements of proteins, which are the most important macromolecules for the functions of humans and animals. Out of the 20 L-amino acids, which are found worldwide in most of the living organisms, L-lysine is one of the nine essential amino acids for human and animal nutrition \hyperref[b6]{(Anastassiadis, 2007)}.\par
Neuro-dynamic programming (NDP) is proposed as an alternative to alleviate the "curse of dimensionality" of the Dynamic programming (DP). The term NDP expresses the reliance of the methods, described in this article with respect to both the DP and the neural network concepts. The term reinforcement learning is also used in the artificial intelligence community where the methods originated from. Using common artificial intelligence terms, the methods help the systems "learn how to make good decisions by observing their own behavior and use built-in mechanisms for improving their actions through a reinforcement mechanism" \hyperref[b0]{(Bertsekas \& Tsitsiklis, 1996)}.\par
The key idea is to use a scoring function to select decisions in complex dynamic systems, arising from a broad variety of applications for engineering design, operations research, resource allocation, finance, etc. This is much similar to a computer chess, where positions are evaluated by means of a scoring function and the move that leads to the position with the best score is chosen. NDP provides a class of systematic methods for computing the appropriate scoring functions using approximation schemes and simulation/evaluation of the system's performance \hyperref[b1]{(Driessens \& Dzeroski, 2004)}.\par
Using common artificial intelligence terms, the methods allow the systems to "learn how to make good decisions by observing their own behavior and use builtin mechanisms for improving their actions through a reinforcement mechanism".\par
In more mathematical meaning "observing their own behavior" relates to simulation and "improving their actions through a reinforcement mechanism" relates to the iterative schemes for improving the quality of approximation of the optimal cost function, the Q-factors or the optimal policy. There has been a gradual realization that the reinforcement learning techniques can be fruitfully motivated and interpreted in terms of classical DP concepts such as the value and policy iteration  {\ref (Barto et. all, 1995;}\hyperref[b3]{Sutton, 1988)}.\par
NDP is a relatively new class of the dynamic programming methods for control and sequential decision making under uncertainty. These methods have the potential of dealing with some problems that were thought to be intractable for a long time due to either a large state space or the lack of an accurate model. They combine ideas from the fields of neural networks, artificial intelligence, cognitive science, simulation, and approximation theory. In recent years L the method has been applied successfully for an optimal control of fermentation process (FP). The literature sources show that the calculating time is significantly reduced, while the desired products quantity is increased  {\ref (Kaisare et. all, 2003;} {\ref Ikova \& Petrov, 2008)}.\par
The aim of this study is to develop optimal feed rate strategy of biotechnological process in L-lysine production using Neuro-dynamic control. 
\section[{II.}]{II.} 
\section[{PROCESS SPECIFICS AND L-LYSINE PRODUCTION MATHEMATICAL MODEL}]{PROCESS SPECIFICS AND L-LYSINE PRODUCTION MATHEMATICAL MODEL}\par
The development of a multi-step biotechnological process requires three steps, comprising of:\par
? Identification and characterization of a suitable biological system (microorganism, biocatalyst).  \hyperref[b6]{(Anastassiadis, 2007)}.\par
In addition to parameters like pH, agitation and aeration rate, air saturation, temperature, dissolved carbon dioxide and foaming, the medium composition is a very important factor highly influencing fermentation processes, which are often a subject of extensive process development and optimization studies. The culture medium has to satisfy the requirements of microbial growth and production in a suitable manner. L-lysine can be produced either using a chemical or a biochemical method, which is economic, even though relatively low yields are obtained during the extraction of L-lysine, requiring specific installations and the use of expensive products. The stereospecificity of amino acids and the steadily increasing L-lysine demand necessitates indispensably their fermentative production (the L isomer) over synthetic processes.\par
The experimental investigations are done in a 15 L bioreactor that is included in an Automatic Control System. The Automatic Control System is flexible and includes control of the following parameters of the process: rotation speed, oxygen partial pressure, temperature, pH, foam level, gas flow rate, flow rates of the main substance. The process is led in the next conditions:? Temperature T=300C; ? pH pH=6.8-7.6; ? pO 2 pO 2 =20-30\%; ? Gas flow rate Q G =60 L h-1; ? Rotation speed n=450 min-1; ? Maximum bioreactor volume 15 L.\par
For the L-lysine fermentation defined media is used which acquires nutrients that require pure growth and essential additives or alternatively undefined media containing natural organic substances such as soybeanhydrolyzate, corn steep liquor, yeast extract or peptone is used. Common fermentation media for Llysine production contain various carbon and nitrogen sources, inorganic ions and trace elements (Fe++, Mn++), amino acids, vitamins (biotin, thiamine-HCl, Nicothin amide) and numerous complex organic compounds. An upper expression of genes is also achieved by optimizing the composition of the media and the culture technique in addition to the physiological and genetic parameters \hyperref[b6]{(Anastassiadis, 2007)}.\par
The model of the fed-batch processes includes the dependences between the concentrations of the basic variables of the process: cell mass concentration (bacteria Brevibacterium flavum), substrate concentration, L-lysine, Threonine concentration and oxygen concentration in the liquid phase. The general scheme of the L-lysine is shown in Figure \hyperref[fig_0]{1}. The mathematical model of the process is based on the mass balance equations as a perfect mixing in the bioreactor is adopted. The model of the process has the following type \hyperref[b6]{(Anastassiadis, 2007)}:(1) (2) X V F X dt dX ? = µ X k X k X k S S V F dt dS in ? µ 7 6 5 ) ( ? ? ? ? = Tr V F X k Tr Tr V F dt dTr in ? ? ? = µ 13 ) ( L L l L C V F X k X k X k C C a k dt dC ? ? ? ? ? = ? µ 16 15 14 * ) ( (3) (4) (5) (6)\par
The specific rate of L-lysine synthesis and specific consumption rate have the following form: (7) \hyperref[b7]{(8)} where:\par
specific rate of L-lysine synthesis, h specific consumption rate of L-lysine, h\par
The initial conditions in the model (  {\ref 1}) -(  {\ref 8}) have the follows values:\par
The model coefficients in (1) -(  {\ref 8}) have the following values: 
\section[{III. NEURO-DYNAMIC OPTIMAL CONTROL OF THE PROCESS}]{III. NEURO-DYNAMIC OPTIMAL CONTROL OF THE PROCESS}\par
The objective of this work is to find the optimal feed flow rate (F(t)) of a fed-batch process, such as the L-lysine production that will raise L-lysine at the end of the process, i.e.: \hyperref[b8]{(9)} where: t 0 -initial time, t f -final time of the fermentation. Therefore, the control objective is to drive the reactor from the low product steady state to the desirable high product rate. It may be considered as a step change in the set point at time t=0 from the low product concentration to the high product concentration steady state.\par
In the systems the decisions are made in stages. The outcome of each decision is not fully predictable but can be anticipated to some extent before the next decision is made. Each decision results in some immediate cost, but it also affects the context in which the future decisions are to be made and thus it affects the cost incurred in future stages. DP provides a mathematical formalization of the tradeoff between the immediate and future costs. Generally, in DP formulations there is a discrete-time dynamic system whose state evolves according to the given transition probabilities that depend on the decision/control u.\par
DP is an elegant way to solve the introduced optimization problem (9). It involves a stagewise calculation of the cost-to-go function to arrive at the solution not just for a specific initial state, but for a general initial state. Once obtained the cost-to-go function, represents a convenient vehicle to obtain the solution for a general state. In very few cases the stagewise optimization to obtain analytically a closedform expression for the cost-to-go function has been solved. The conventional approach to the problem involves gridding the state space, calculating and storing the cost-to-go for each grid points as one marches backward from the first stage to the last. For an infinite horizon problem the number of iteration required for convergence can be very big. Such an approach is seldom practically feasible due to the exponential growth of the computation with respect to the state dimension. Unfortunately, from the very beginning it was apparent that an increase of the dimensionality of the problem, i.e. an addition of reservoirs, caused an exponential increase in the time required to find a solution. This is referred to as the "curse of dimensionality", which must be removed so that this approach can find a widespread use.\par
NDP aims to develop a methodological foundation for combining dynamic programming, compact representations, and simulation to provide the basis for a rational approach to complex stochastic decision problems \hyperref[b0]{(Bertsekas \& Tsitsiklis, 1996;} {\ref Kaisare et. all, 2003)}.\par
Two fundamental DP algorithms, policy iteration and value iteration, are the starting points for the NDP methodology. The most straightforward adaptation of the policy iteration method operates as follows: we start with a given policy (a rule for choosing a decision u at each possible state i), and we approximately evaluate the cost of that policy (as a function of the current state) by least-squares-fitting a scoring function to the resultsX dt dL ? = F dt dV = ( )( )( ) [ ] L L C k S S k Tr k C Tr k + ? + + = 4 0 3 2 1 µ ( )( )( )( ) [ ] L L L C k C k S k S k C S k + + + + = 12 11 10 9 8 ? µ - -1 ; ? - -1 ; X -\par
biomass concentration, g l -; L -L-lysine concentration, g l -1 ; S -glucose concentration, g l -1 ; V -working liquid volume, l; F -feed flow rate, l h -1 ; Tr -Threonine concentration, mg l -1 ; t -process time, h; C L -dissolved oxygen concentration, g l -1 ; C* -equilibrium dissolved oxygen concentration, g l -1 ; S in -input feed substrate concentration, g l -1 ; Tr in -input feed Threonine concentration, g l -1 ; k l a -volumetric liquid mass transfer coefficient, h -1 . X(0) = X 0 = 3.00 g l -1 ; S(0) = S 0 = S i = 100.00 g l -1 ; Tr(0) = Tr 0 = Tr in = 100.00 mg l -1 ; L(0) = 0.00 g l -1 ; C L (0) = C * = C 0 = 6.1x10 -3 g l -1 ; V(0) = V 0 = 10.00 l. k 1 = 20.8, k 2 = 42.0, k 3 = 28.0, k 4 = 1.1, k 5 = 1.01, k 6 = 0.07, k 7 = 0.51, k 8 = 62.0, k 9 = 28.0, k 10 = 37.0, k 11 = 4.0, k 12 = 0.12, k 13 = 6.10, k 14 = 448.0, k 15 = 22.0, k 16 = 209.0, k l a = 120.? = f t t dt t V t L Q 0 ) ( ) ( max\par
u of many simulated system trajectories using that policy. A new policy is then defined by minimization in Bellman's equation where the optimal cost is replaced by the calculated scoring function and the process is repeated. This type of algorithm typically generates a sequence of policies that eventually oscillates in a surrounding of an optimal policy. The resulting deviation from optimality depends on a variety of factors, principal among which is the ability of the architecture of scoring function to accurately approximate the cost functions of the various policies.\par
NDP uses simulated process data received under suboptimal policies to fit an approximate cost-togo function -generally by fitting artificial network. With the value iteration approach NDP the initial approximate cost-to-go function in the future was improved by an iteration procedure based on Bellman equation. In this way the simulation role has two points. First, by simulation the process under a reasonably chosen suboptimal policy and all possible operating parameters it provides set data points that define the relevant "working" region in the state space. Second, the simulation provides the cost-to-go value under the suboptimal policy for each state visited, which iteration of the Bellman equation can be initialed with  {\ref (Kaisare et. all, 2003)}.\par
In this paper we will demonstrate NDP approach not only for reducing the computational demand, but also for improving the controller performance through the use of the cost-to-go approximator. A neural network is chosen as an approximator to obtain cost-to-go as a function of system states. While a properly trained neural network has good interpolation capabilities, one may not be used to extrapolate over the regions of state space that are not covered during its training. Extrapolation by neural network results in deteriorated performance of the controller.\par
The policy improvement theorem states that a new policy that is greedy (a greedy policy is one whose current cost is the least) with respect to the cost-to-go function of the original policy is as good as or better than the original policy.\par
When the new policy is as good as the original policy the above equation becomes the same as Bellman equation.\par
The relevant regions of the state space are identified by simulation of NDP control and the initial suboptimal cost-to-go function is calculated from the simulation data. In this survey a functional approximator is used to interpolate between this data. The improvement is obtained through the iteration of the Bellman equation. When the iteration converge this offline computed cost-to-go function can be used for an on-line optimal control calculation for the bioreactor \hyperref[b10]{(Xiong \& Zhang, 2005)}.\par
NDP uses neural network approximations for the approximation of cost-to-go function. The cost-to-go function was not used to generate an explicit control law; instead, it was used in an on-line optimization to reduce the large (or infinite) horizon problem to a relatively short horizon problem. The method was found to be robust to approximation errors. Both deterministic (step changes in kinetic parameters) and stochastic problems (random variations in kinetic parameters and feed composition) were explored Lee J.M., \& Lee, J. H., 2009; Tosukhowong \& Lee J. H., 2009). 
\section[{a) NDP algorithm}]{a) NDP algorithm}\par
The following notations are used for description of the algorithm:\par
The general simulation-approximation scheme involves computation of the converged cost-to-go approximation off-line. The architecture of the scheme is shown in Figure  {\ref 2} The simulation-based approach involves computation of the converged profit-to-go approximation off-line. The following steps describe the general procedure of NDP algorithm: 1. Performing of simulations of the process with chosen suboptimal policies under all representative operating conditions. Starting with a given policy (a rule for choosing a decision u at each possible state i), and approximately evaluate the cost of that policy (as a function of the current state) by least-squaresfitting a scoring function to the results of the many simulated system trajectories using that policy. 2. Calculation of the -horizon cost-to-go for each state visited during the simulation, using the simulation data. The solution of one-stage-ahead cost plus cost-to-go problem results in the improving of the cost values. Cost-to-go is the sum of the single state cost from the next point to the end of the horizon. 3. The deviation, which is a result of the optimality, depends on a variety of factors, principal among which is the ability of the architecture of Bellman function to approximate accurately the cost functions of the various policies. 4. A new policy is then defined by minimizing Bellman's equation where the optimal cost is replaced by the calculated scoring function and the process repeats. This type of algorithm typically generates a sequence of policies that eventually oscillate in a surrdounding of an optimal policy. 5. Fitting a neural network function approximator to the data to approximate the cost-to-go function as a smooth function of the states. 6. As described above the improved costs are again fitted to a neural network, to obtain subsequent ? iterations of Bellman functions, and so on ..., until convergence is accomplished. 7. Policy update may sometimes be necessary to increase the coverage of the state space. In this case more suboptimal simulations with the updated policy are used to increase the coverage or the number of the data points in certain region of the state space.\par
The NDP algorithm block-scheme is shown in Fig.  {\ref 2}.\par
Fig.  {\ref 2} : NDP algorithm block-scheme.\par
Take into consideration that when starting with a fairly good approximation of the cost-to-go (which has to be a result of using a good suboptimal policy), the cost iteration has to converge fairly fast -faster than the conventional stagewise cost-to-go calculation.\par
The next values of F are examined 0.2, 0.4, 0.5, 0.7, that can cover the possible rang of variations. The bioreactor was started at three different W(0) values for each of the parameter values around the low product yield steady state.\par
A functional approximation relating cost-to-go with augmented state was obtained by the neural network -with five hidden nodes, six input nodes and two output nodes. The neural network presented a good fit with a mean error of 10-3 after training for 1000 epoch.\par
Improvement of the cost-to-go is obtained through the iterations of the Bellman equation. This method is known as a value iteration (or value iteration). The solution of the one-stage-ahead cost plus cost-togo problem, results in the improvement of the cost values. The improved prices were again fitted to the neural network, described above to obtain subsequent iterations of Belman function and so on ..., until they are converged. Cost is said to be "converged" if the sum of the absolute error is less than 5\% of the maximum cost. The cost is converged in 7 iterations for our system.\par
The converged cost-to-go function from above was used for solving the one-stage-ahead problem.\par
The optimal value of feed flow rate before and after optimization is shown in Fig. 3. The L-lysine production before and after optimization is shown in Fig. 4. Fig. 4 shows the increase of the L-lysine after optimization by 39.41\%.  The results show that after 48th the process stands still and it continuing is economically disadvantageous. It becomes clear that after 48th the process goes into a steady state. Therefore, the fixed right end for 48 hours is appropriate.\par
In this optimization problem the time is discredited in six hours. It is assumed that this is a step of discretization of this process in terms of features and well-known computational difficulties.\par
IV. 
\section[{CONCLUSION}]{CONCLUSION}\par
An approach for the optimal control of fermentation processes for a L-lysine fed-batch fermentation is developed for searching an optimal feed rate strategy using Neuro-dynamic control. It is proposed as a method for alleviation of the "curse of dimensionally" of DP.\par
The conventional approach to solving an optimization problem with DP method involves gridding of the state space, solving the optimization for each grid point and performing the stagewise optimization until convergence. Exhaustive sampling of state space can be avoided by identifying relevant regions of the state space by simulation under judiciously chosen suboptimal policies, which is presented using NDP methods with the help of a neural network for functional approximator.\par
The results show that the L-lysine quantity is highly raised at the end of the process which is the desired criterion for process quality. The result shows that NDP is a convenient and easy to use application method for optimal control. The approach is particularly simple to implement and it should be used for on-line implementation, after necessary additional training of the relevant neural network is obtained. L-lysine before L-lysine after \begin{figure}[htbp]
\noindent\textbf{1}\includegraphics[]{image-2.png}
\caption{\label{fig_0}Fig. 1 :}\end{figure}
 \begin{figure}[htbp]
\noindent\textbf{1}\includegraphics[]{image-3.png}
\caption{\label{fig_1}. Step 1 ,}\end{figure}
   \begin{figure}[htbp]
\noindent\textbf{} \par 
\begin{longtable}{}
\end{longtable} \par
 
\caption{\label{tab_1}}\end{figure}
 			\footnote{© 2011 Global Journals Inc. (US) L-lysine Neuro-Dynamic Optimal Control} 			\footnote{December} 			\footnote{DecemberVolume XI Issue IV Version I} 		 		\backmatter  			  				\begin{bibitemlist}{1}
\bibitem[Soni ()]{b7}\label{b7} 	 		\textit{A multi-scale approach to fed-batch bioreactor control},  		 			A Soni 		.  		2002. Pennsylvania: University of Pittsburgh Press.  	 
\bibitem[Lee et al. ()]{b9}\label{b9} 	 		‘Choice of approximator and design of penalty function for an approximate dynamic programming based control approach’.  		 			J M Lee 		,  		 			N S Kaisare 		,  		 			J H Lee 		.  	 	 		\textit{Journal of Process Control}  		2006. 16 p. .  	 
\bibitem[Ilkova and Petrov ()]{b5}\label{b5} 	 		\textit{Dynamic and neurodynamic optimization of a fed-batch fermentation process. Lecture Notes in Artifical Intelligence},  		 			T Ilkova 		,  		 			M Petrov 		.  		2008. 5253 p. .  	 
\bibitem[Vlachos et al. ()]{b8}\label{b8} 	 		‘Hierarchical Multiscale model-based design of experiments, catalysts and reactors for fuel processing’.  		 			D G Vlachos 		,  		 			A B Mhadeshwar 		,  		 			N S Kaisare 		.  	 	 		\textit{Computer \& Chemical Engineering}  		2006. 30 p. .  	 
\bibitem[Driessens and Dzeroski ()]{b1}\label{b1} 	 		‘Integrating guidance intro relational reinforcement learning’.  		 			K Driessens 		,  		 			S Dzeroski 		.  	 	 		\textit{Machine Learning},  				2004. 57 p. .  	 
\bibitem[Anastassiadis ()]{b6}\label{b6} 	 		‘L-Lysine fermentation’.  		 			S Anastassiadis 		.  	 	 		\textit{Recent patents on biotechnology}  		2007. 1 p. .  	 
\bibitem[Sutton ()]{b3}\label{b3} 	 		‘Learning to predict by the methods of temporal differences’.  		 			R S Sutton 		.  	 	 		\textit{Machine Learning},  				1988. 3 p. .  	 
\bibitem[Xiong and Zhang ()]{b10}\label{b10} 	 		‘Neural Network model-based on-line re-optimisation control of fedbatch processes using a modified iterative dynamic programming algorithm’.  		 			Zh Xiong 		,  		 			J Zhang 		.  	 	 		\textit{Chemical Engineering and Processing}  		2005. 44 p. .  	 
\bibitem[Bertsekas and Tsitsiklis ()]{b0}\label{b0} 	 		\textit{Neuro-dynamic programming},  		 			D Bertsekas 		,  		 			J Tsitsiklis 		.  		1996. Massachusetts: Athena Scientific Belmont.  	 
\bibitem[Barto et al. ()]{b2}\label{b2} 	 		‘Real-time learning and control using asynchronous dynamic programming’.  		 			A G Barto 		,  		 			S J \&bradtke 		,  		 			S P Singh 		.  	 	 		\textit{Artificial Intelligence}  		1995. 72 p. .  	 
\bibitem[Kaisare et al. ()]{b4}\label{b4} 	 		‘Simulation based strategy for nonlinear optimal control: application to a microbial cell reactor’.  		 			N S Kaisare 		,  		 			J M Lee 		,  		 			J H Lee 		.  	 	 		\textit{International Journal of Robust and Nonlinear Control}  		2003. 13 p. .  	 
\end{bibitemlist}
 			 		 	 
\end{document}
