(************** Content-type: application/mathematica ************** CreatedBy='Mathematica 5.0' Mathematica-Compatible Notebook This notebook can be used with any Mathematica-compatible application, such as Mathematica, MathReader or Publicon. The data for the notebook starts with the line containing stars above. To get the notebook into a Mathematica-compatible application, do one of the following: * Save the data starting with the line of stars above into a file with a name ending in .nb, then open the file inside the application; * Copy the data starting with the line of stars above to the clipboard, then use the Paste menu command inside the application. Data for notebooks contains only printable 7-bit ASCII and can be sent directly in email or through ftp in text mode. Newlines can be CR, LF or CRLF (Unix, Macintosh or MS-DOS style). NOTE: If you modify the data for this notebook not in a Mathematica- compatible application, you must delete the line below containing the word CacheID, otherwise Mathematica-compatible applications may try to use invalid cache data. For more information on notebooks and Mathematica-compatible applications, contact Wolfram Research: web: http://www.wolfram.com email: info@wolfram.com phone: +1-217-398-0700 (U.S.) Notebook reader applications are available free of charge from Wolfram Research. *******************************************************************) (*CacheID: 232*) (*NotebookFileLineBreakTest NotebookFileLineBreakTest*) (*NotebookOptionsPosition[ 505892, 14558]*) (*NotebookOutlinePosition[ 522643, 14977]*) (* CellTagsIndexPosition[ 519010, 14885]*) (*WindowFrame->Normal*) Notebook[{ Cell[CellGroupData[{ Cell["Notes", "Section 1"], Cell[CellGroupData[{ Cell["Editorial Changes", "Subsection"], Cell["American spelling changed to British spelling.", "Text"], Cell["Equation numbers (section.number) changed to (number).", "Text"], Cell["Line breaks in equations removed where they are not needed.", "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["PROBLEM", ButtonData:>"Eq:EuclideanSplit", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ \(TraditionalForm\`\(\(||\)\(\[CenterEllipsis]\)\( || \^2\)\)\)]], " on the righthand side has to be non-spanning because a bug prevents the \ spanning version from working." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change1", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change2", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[Cell[TextData[Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], StyleBox["/", FontSlant->"Italic"], RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], "\[Prime]"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "=", "0"}], TraditionalForm]]]]], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold"]}]], "0"], "\[Prime]"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change3", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change4", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[Cell[TextData[Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], StyleBox["/", FontSlant->"Italic"], RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], "\[Prime]"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], "=", "0"}], TraditionalForm]]]]], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold"]}]], "0"], "\[Prime]"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change5", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[Cell[TextData[Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], StyleBox["/", FontSlant->"Italic"], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], "=", "0"}], TraditionalForm]]]]], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change6", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox[ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontSlant->"Plain"], StyleBox["D", FontSlant->"Italic"]}]], StyleBox["0", FontSlant->"Plain"]], FontSlant->"Italic"], "(", StyleBox["a", FontSlant->"Italic"], StyleBox[")", FontSlant->"Italic"]}], StyleBox["/", FontSlant->"Italic"], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox[ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontSlant->"Plain"], StyleBox["D", FontSlant->"Italic"]}]], StyleBox["0", FontSlant->"Plain"]], FontSlant->"Italic"], "(", StyleBox["b", FontSlant->"Italic"], StyleBox[")", FontSlant->"Italic"]}], StyleBox["/", FontSlant->"Italic"], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{ StyleBox[ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontSlant->"Plain"], StyleBox["D", FontSlant->"Italic"]}]], StyleBox["0", FontSlant->"Plain"]], FontSlant->"Italic"], "(", StyleBox["a", FontSlant->"Italic"], StyleBox[")", FontSlant->"Italic"]}], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{ StyleBox[ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontSlant->"Plain"], StyleBox["D", FontSlant->"Italic"]}]], StyleBox["0", FontSlant->"Plain"]], FontSlant->"Italic"], "(", StyleBox["b", FontSlant->"Italic"], StyleBox[")", FontSlant->"Italic"]}], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]], TraditionalForm]]], "\", and \"", Cell[BoxData[ FormBox[Cell[TextData[Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], StyleBox["/", FontSlant->"Italic"], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], "=", "0"}], TraditionalForm]]]]], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change7", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"\[Epsilon]", "/", StyleBox[ RowBox[{"\[Delta]", StyleBox["V", FontSlant->"Italic"]}]]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox["\[Epsilon]", StyleBox[ RowBox[{"\[Delta]", StyleBox["V", FontSlant->"Italic"]}]]], TraditionalForm]]], "\", and \"", Cell[BoxData[ FormBox[ RowBox[{"1", "/", StyleBox[ RowBox[{"\[Delta]", StyleBox["L", FontSlant->"Italic"]}]]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox["1", StyleBox[ RowBox[{"\[Delta]", StyleBox["L", FontSlant->"Italic"]}]]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change8", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"\[Delta]", RowBox[{\(D\&_\), "/", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Italic"]}]], \(1, 0\)], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[\(\[Delta] D\&_\), RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Italic"]}]], \(1, 0\)], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:BatchTrainSplit", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["arg", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]], " ", "min"}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[GridBox[{ {\(arg\ min\)}, { StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]} }], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:OnLineTrainVQ2Discrete", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], RowBox[{"0", ",", SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"]}]]}], ")"}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], RowBox[{"0", ",", SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"]}]]}], TraditionalForm]]], "\"." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["A Bayesian Analysis of Self-Organising Maps", "Title"], Cell["\<\ Stephen P. Luttrell Adaptive Systems Theory Section, Defence Research Agency St Andrews Rd., Malvern, Worcestershire, WR14 3PS, United Kingdom\ \>", "Author"], Cell["\<\ This paper appeared in Neural Computation, 1994, vol. 6, pp. 767-794. Communicated by Graeme Mitchison Received May 19, 1993; accepted September 7, 1993.\ \>", "Text"], Cell["\[Copyright] 1994 British Crown Copyright/DRA", "Text"], Cell[TextData[StyleBox["In this paper Bayesian methods are used to analyse \ some of the properties of a special type of Markov chain. The forward \ transitions through the chain are followed by inverse transitions (using \ Bayes' theorem) backwards through a copy of the same chain; this will be \ called a folded Markov chain. If an appropriately defined Euclidean error \ (between the original input and its \"reconstruction\" via Bayes' theorem) is \ minimised with respect to the choice of Markov chain transition \ probabilities, then the familiar theories of both vector quantisers and \ self-organising maps emerge. This approach is also used to derive the theory \ of self-supervision, in which the higher layers of a multi-layer network \ supervise the lower layers, even though overall there is no external \ teacher.", FontWeight->"Bold"]], "Abstract"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Introduction" }], "Section 1", CellTags->"Sect:Introduction"], Cell["\<\ A self-organising map (SOM) is an adaptive function that transforms (or maps) \ from an input vector space to an output vector space, where the adaptation is \ driven entirely by signals derived from the input space. In the context of \ neural networks an SOM would therefore be realised as an unsupervised network \ whose training algorithm acted to minimise a suitably defined error function \ in the network's input space. The aim of this paper is to develop a \ theoretical framework that unifies several different strands of unsupervised \ network theory, and for this purpose it is best to develop the theory anew, \ rather than to build a hybrid theory out of an assortment of existing \ theories.\ \>", "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Variational Formulation of SOM Optimisation" }], "Subsection"], Cell[TextData[{ "In order to create a theoretically clean framework it is necessary to \ express the optimisation of an SOM in terms of a variational principle. Thus \ define a scalar functional ", Cell[BoxData[ \(TraditionalForm\`D\)]], " as follows" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"D", "\[Congruent]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"P", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{"y", "(", StyleBox["x", FontWeight->"Bold"], ")"}], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective1"], Cell[TextData[{ "where the vectors ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " sit in the input and output spaces respectively, and the functions ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], " transform from ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], "-space to ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], "-space and vice-versa respectively. These are the functions that must be \ optimised so as to minimise the value of ", Cell[BoxData[ \(TraditionalForm\`D\)]], ". The scalar function ", Cell[BoxData[ FormBox[ RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}], TraditionalForm]]], " contributes to the functional ", Cell[BoxData[ \(TraditionalForm\`D\)]], " an amount that depends on the currently selected input vector ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], ", and on the functions ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], ". The function ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], " is a probability density (or measure) that weights the ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], " integral non-uniformly. In practice ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], " might be used to represent the relative frequency of occurrence of each \ input vector ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], " in a representative set of input vectors, in which case ", Cell[BoxData[ \(TraditionalForm\`D\)]], " would be the average of ", Cell[BoxData[ FormBox[ RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}], TraditionalForm]]], " over that set of vectors." }], "Text"], Cell[TextData[{ "In ", ButtonBox["equation", ButtonData:>"Eq:Objective1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:Objective1"], " the functional dependence of ", Cell[BoxData[ FormBox[ RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}], TraditionalForm]]], " can be simplified to ", Cell[BoxData[ FormBox[ RowBox[{"d", RowBox[{"{", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "[", RowBox[{ StyleBox["y", FontWeight->"Bold"], StyleBox["(", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold"], ")"}], "]"}]}], "}"}]}], TraditionalForm]]], ", so ", Cell[BoxData[ \(TraditionalForm\`D\)]], " simplifies to" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"D", "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"P", "(", StyleBox["x", FontWeight->"Bold"], ")"}], "d", RowBox[{"{", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "[", RowBox[{ StyleBox["y", FontWeight->"Bold"], StyleBox["(", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold"], ")"}], "]"}]}], "}"}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective2"], Cell[TextData[{ "This is possible because in ", Cell[BoxData[ FormBox[ RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}], TraditionalForm]]], " the vector ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " in the function ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], " is a placeholder that receives the output of the function ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], ". The variational problem is then to find functions ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ AdjustmentBox["y", BoxMargins->{{0.25, -0.25}, {0, 0}}], FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], " that minimise ", Cell[BoxData[ \(TraditionalForm\`D\)]], " as defined in ", ButtonBox["equation", ButtonData:>"Eq:Objective2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:Objective2"], "." }], "Text"], Cell[TextData[{ ButtonBox["Equation", ButtonData:>"Eq:Objective2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:Objective2"], " can readily be extended to include the effects of an additive noise \ process acting in ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], "-space. Thus ", Cell[BoxData[ \(TraditionalForm\`D\)]], " becomes" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"D", "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"P", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["n", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"\[Pi]", "(", StyleBox["n", FontWeight->"Bold"], ")"}], "d", RowBox[{"{", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "[", RowBox[{ RowBox[{ StyleBox["y", FontWeight->"Bold"], StyleBox["(", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold"], ")"}], "+", StyleBox["n", FontWeight->"Bold"]}], "]"}]}], "}"}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective3"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["\[Pi]", FontWeight->"Bold"], "(", StyleBox["n", FontWeight->"Bold"], ")"}], TraditionalForm]]], " is the probability density of the noise vector ", Cell[BoxData[ FormBox[ StyleBox["n", FontWeight->"Bold"], TraditionalForm]]], ". The variational problem is then to find functions ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], " that minimise this augmented expression for ", Cell[BoxData[ \(TraditionalForm\`D\)]], ". A side effect of including the noise process is that the information \ that is stored in the output vector ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " (produced by the action of ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], " transforming the input vector ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], ") is represented in such a way that it is robust with respect to the \ damaging effects of the noise process." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Variational Principle versus Unsupervised Network Notation" }], "Subsection"], Cell[TextData[{ "By making the replacement ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"d", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"]}], ")"}], "=", RowBox[{"||", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"]}], \( || \^2\)}]}], TraditionalForm]]], " (i.e., a Euclidean distance) in ", ButtonBox["equation", ButtonData:>"Eq:Objective3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:Objective3"], " this variational formulation can be shown to lead to a type of SOM that \ is similar to, but not precisely the same as, the unsupervised network that \ is known as the Kohonen map (see ", ButtonBox["Kohonen 1984", ButtonData:>"Ref:Kohonen1984", ButtonStyle->"Hyperlink"], ") with the noise probability density ", Cell[BoxData[ FormBox[ RowBox[{"\[Pi]", RowBox[{"(", StyleBox["n", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " playing the role of the SOM neighbourhood function. This type of analysis \ was also introduced in a non-neural context to design an optimal vector \ quantiser (VQ) codebook for encoding data for transmission along a noisy \ channel (", ButtonBox["Kumazawa ", ButtonData:>"Ref:KumazawaKasaharaNamekawa1984", ButtonStyle->"Hyperlink"], StyleBox[ButtonBox["et al.", ButtonData:>"Ref:KumazawaKasaharaNamekawa1984", ButtonStyle->"Hyperlink"], FontSlant->"Italic"], ButtonBox[" 1984", ButtonData:>"Ref:KumazawaKasaharaNamekawa1984", ButtonStyle->"Hyperlink"], "; ", ButtonBox["Farvardin 1990", ButtonData:>"Ref:Farvardin1990", ButtonStyle->"Hyperlink"], "; ", ButtonBox["Farvardin and Vaishampayan 1991", ButtonData:>"Ref:FarvardinVaishampayan1991", ButtonStyle->"Hyperlink"], ")." }], "Text"], Cell["\<\ The above variational principle can be related to a corresponding \ unsupervised network as shown in the table below.\ \>", "Text"], Cell[BoxData[ FormBox[GridBox[{ {"Term", \(Variational\ principle\), \(Unsupervised\ network\)}, { StyleBox["x", FontWeight->"Bold"], \(Input\ vector\), \(Training/ test\ vector\)}, { RowBox[{ StyleBox["y", FontWeight->"Bold"], RowBox[{"(", StyleBox["x", FontWeight->"Bold"], ")"}]}], \(Input\ to\ output\ transformation\), \(Encoding\ \ prescription\)}, { RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}], \(Output\ to\ input\ transformation\), \(Reference\ \ vector\)}, { RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}], "Function", \(Error\ function\)}, { RowBox[{"P", RowBox[{"(", StyleBox["x", FontWeight->"Bold"], ")"}]}], \(Integration\ measure\), \(Probability\ density\ of\ \ training/test\ vectors\)}, {"D", "Functional", \(Average\ error\ over\ the\ training/test\ set\)} }, GridFrame->True, RowLines->True, ColumnLines->True], TraditionalForm]], "NumberedTable"], Cell["Variational Principle and Unsupervised Networks.", "Caption"], Cell[TextData[{ "Note that in practice the output ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " of an unsupervised network is usually the index of the \"winning node\", \ which is a discrete-valued quantity. Whether the output is continuous or \ discrete is unimportant to the variational approach, but in this paper a \ continuum notation is used because it is more compact." }], "Text"], Cell[TextData[{ "The basic derivations of the variational formulation are in Luttrell (", ButtonBox["1989a", ButtonData:>"Ref:Luttrell1989a", ButtonStyle->"Hyperlink"], ",", ButtonBox["c", ButtonData:>"Ref:Luttrell1989c", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1990", ButtonData:>"Ref:Luttrell1990", ButtonStyle->"Hyperlink"], "), a simple application to time series compression is in ", ButtonBox["Luttrell (1989b)", ButtonData:>"Ref:Luttrell1989b", ButtonStyle->"Hyperlink"], ", the compression of synthetic aperture radar images is in ", ButtonBox["Luttrell (1989d)", ButtonData:>"Ref:Luttrell1989d", ButtonStyle->"Hyperlink"], ", an analysis of the density of reference vectors is in ", ButtonBox["Luttrell (1991a)", ButtonData:>"Ref:Luttrell1991a", ButtonStyle->"Hyperlink"], ", and an extension of unsupervised networks to multi-layer networks in \ which higher layers supervise lower layers (self-supervised networks) is in \ Luttrell (", ButtonBox["1991b", ButtonData:>"Ref:Luttrell1991b", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1992", ButtonData:>"Ref:Luttrell1992", ButtonStyle->"Hyperlink"], ")." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Encoding Prescriptions" }], "Subsection"], Cell[TextData[{ "It has been noted (Luttrell ", ButtonBox["1989a", ButtonData:>"Ref:Luttrell1989a", ButtonStyle->"Hyperlink"], ",", ButtonBox["c", ButtonData:>"Ref:Luttrell1989c", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1990", ButtonData:>"Ref:Luttrell1990", ButtonStyle->"Hyperlink"], "; ", ButtonBox["Favardin and Vaishampayan 1991", ButtonData:>"Ref:FarvardinVaishampayan1991", ButtonStyle->"Hyperlink"], ") that the above Euclidean error function is not minimised when the \ nearest neighbour encoding prescription is used. Exact minimisation requires \ a new type of winner-take-all encoding prescription to be used; the nearest \ neighbour prescription must be replaced by the so-called minimum distortion \ prescription, in which the choice of winner is influenced by the reference \ vectors to which it is connected by the SOM neighbourhood function. ", ButtonBox["Figure", ButtonData:>"Fig:MinimumDistortionEncoding", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:MinimumDistortionEncoding"], " shows graphically how nearest neighbour and minimum distortion encoding \ are related." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig1.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:MinimumDistortionEncoding"], Cell[TextData[{ "Assume that the input vector ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], " and the function ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " are such that the error function ", Cell[BoxData[ FormBox[ RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}], TraditionalForm]]], " has the ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " dependence shown above by the solid curve. There is a well-defined \ minimum, and the shape of the minimum is skewed so that ", Cell[BoxData[ FormBox[ RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}], TraditionalForm]]], " increases more rapidly to the right than to the left of the minimum. \ Assume that the SOM neighbourhood function ", Cell[BoxData[ FormBox[ RowBox[{"\[Pi]", "[", RowBox[{ StyleBox["y", FontWeight->"Bold"], "-", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}]}], "]"}], TraditionalForm]]], " is the symmetric function denoted by the dashed curve. The error function \ must be averaged over ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " (weighted by the SOM neighbourhood function) to determine the expected \ error for the particular choice of ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], ", ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ AdjustmentBox["y", BoxMargins->{{0.25, -0.25}, {0, 0}}], FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], ". In effect, ", Cell[BoxData[ FormBox[ RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}], TraditionalForm]]], " is convolved with ", Cell[BoxData[ FormBox[ RowBox[{"\[Pi]", "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], " to produce a smeared error function. The encoding prescription is one's \ choice of ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ AdjustmentBox["y", BoxMargins->{{0.25, -0.25}, {0, 0}}], FontWeight->"Bold"], RowBox[{"(", StyleBox["x", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], ", and the optimum choice is not the minimum of the original error function \ with respect to ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " (nearest neighbour encoding), but rather the minimum of the smeared error \ function with respect to ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " (minimum distortion encoding). Because of the skewed ", Cell[BoxData[ FormBox[ RowBox[{"d", "[", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "]"}], TraditionalForm]]], " the minimum distortion encoding of ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], " is thus displaced a little to the left of the nearest neighbour \ encoding." }], "Caption"], Cell[TextData[{ "The minimum distortion prescription was used in ", ButtonBox["Luttrell (1991a)", ButtonData:>"Ref:Luttrell1991a", ButtonStyle->"Hyperlink"], " to study the equilibrium density of reference vectors in a 1-dimensional \ input space, where it was reported that the result was insensitive to the \ choice of neighbourhood function used in the SOM, provided that it was a \ monotonically decreasing symmetric function. The corresponding results using \ a standard SOM with the nearest neighbour prescription appeared in ", ButtonBox["Ritter (1991)", ButtonData:>"Ref:Ritter1991", ButtonStyle->"Hyperlink"], ", where it was shown that a neighbourhood-sensitive density of reference \ vectors emerged." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Self-Supervised Multi-Layer SOMs" }], "Subsection"], Cell[TextData[{ "Hierarchical multi-layer SOMs were studied in Luttrell (", ButtonBox["1991b", ButtonData:>"Ref:Luttrell1991b", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1992", ButtonData:>"Ref:Luttrell1992", ButtonStyle->"Hyperlink"], "). For instance, a 2-layer version of this type of network splits the \ input space into a number of lower dimensional subspaces, and trains a SOM in \ each subspace. Simultaneously, another SOM is trained on the outputs of the \ above SOMs to produce the final network output. This approach can be cascaded \ to more stages if required. This type of multi-layer SOM can be refined by \ introducing an error function which measures the average Euclidean error \ between the input vector and a reference vector that is constructed from the \ final network output (Luttrell ", ButtonBox["1991b", ButtonData:>"Ref:Luttrell1991b", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1992", ButtonData:>"Ref:Luttrell1992", ButtonStyle->"Hyperlink"], "). This forces the optimisation of the layers to be tied together in such \ a way that they have to be trained co-operatively. For instance, in a 2-layer \ network the outputs from the SOMs in the first layer have to be matched to \ the capabilities of the SOM in the second layer; this is achieved by sending \ back-propagation signals from the second layer to the first layer. However, \ these back-propagation signals are not derived from an external supervisor, \ so this type of training algorithm is called \"self-supervised\"." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Purpose of this Paper" }], "Subsection"], Cell[TextData[{ "The purpose of this paper is to embed the above variational formulation of \ SOMs in a more general framework in which the transformations ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], " are replaced by probabilistic transformations ", Cell[BoxData[ FormBox[ RowBox[{\(P\_1\), "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "|", StyleBox["x", FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(P\_2\), "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "|", StyleBox["y", FontWeight->"Bold"]}], ")"}], TraditionalForm]]], ". ", ButtonBox["Equation", ButtonData:>"Eq:Objective2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:Objective2"], " then becomes" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"D", "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"P", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["y", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], RowBox[{\(P\_2\), "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "|", StyleBox["y", FontWeight->"Bold"]}], ")"}], RowBox[{\(P\_1\), "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "|", StyleBox["x", FontWeight->"Bold"]}], ")"}], RowBox[{"d", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"]}], ")"}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective4"], Cell[TextData[{ "The integration over ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ StyleBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " performs a weighted average of ", Cell[BoxData[ FormBox[ RowBox[{"d", RowBox[{"{", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "[", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}], "]"}]}], "}"}]}], TraditionalForm]]], " over a variety of alternative transformations ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["y", FontWeight->"Bold"], RowBox[{"(", StyleBox["x", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], ", rather than just a single pair of transformations as would have been the \ case in the basic variational formulation. Similarly ", ButtonBox["equation", ButtonData:>"Eq:Objective3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:Objective3"], " becomes" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"D", "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"P", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["n", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"\[Pi]", "(", StyleBox["n", FontWeight->"Bold"], ")"}], RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["y", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], RowBox[{\(P\_2\), "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "|", RowBox[{ StyleBox["y", FontWeight->"Bold"], "+", StyleBox["n", FontWeight->"Bold"]}]}], ")"}], RowBox[{\(P\_1\), "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "|", StyleBox["x", FontWeight->"Bold"]}], ")"}], RowBox[{"d", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"]}], ")"}]}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective5"], Cell["\<\ This \"sum over alternatives\" is useful for a number of reasons.\ \>", "Text"], Cell["\<\ 1. Theoretical manipulations are easier to perform on \"soft\" probabilities \ than on \"hard\" deterministic functions.\ \>", "Text"], Cell["\<\ 2. The probabilistic approach lends itself well to a simulated annealing \ approach in numerical simulations.\ \>", "Text"], Cell[TextData[{ "3. Contact with standard results can be made by making the replacement ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"d", RowBox[{"(", RowBox[{ StyleBox["x", FontWeight->"Bold"], ",", SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"]}], ")"}]}], "=", SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"]}], "\[RightDoubleBracketingBar]"}], "2"]}], TraditionalForm]]], " (i.e. a Euclidean distance) in ", ButtonBox["equation", ButtonData:>"Eq:Objective5", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:Objective5"], ". This leads to a probabilistic generalisation of standard SOM theory." }], "Text"], Cell[TextData[{ "4. If a single pair of transformations ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["y", FontWeight->"Bold"], RowBox[{"(", StyleBox["x", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], " was being used, but one was uncertain about which particular pair, then a \ probabilistic formulation would be necessary." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Structure of this Paper" }], "Subsection"], Cell[TextData[{ "The principal new result in this paper is a Bayesian derivation of the \ properties of SOMs starting from a generalisation of ", ButtonBox["equation", ButtonData:>"Eq:Objective4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:Objective4"], ". Thus a Markov chain of probabilistic transformations of the input vector \ is inverted by sending the (probabilistic) output of the chain back through \ the inverse probabilistic transformations (derived from Bayes' theorem) to \ eventually re-emerge as a (probabilistic) reconstructed version of the input \ vector. This type of structure will be called a folded Markov chain (FMC). \ This FMC is then optimised by minimising the average Euclidean error between \ the input vector and its reconstruction. Various constraints can be placed on \ this optimisation. For instance, if a 2-stage FMC (i.e. 2 stages of \ probabilistic transformation) is considered, and only its first stage is \ optimised, then the theory of SOMs emerges (as in Luttrell (", ButtonBox["1989a", ButtonData:>"Ref:Luttrell1989a", ButtonStyle->"Hyperlink"], ",", ButtonBox["c", ButtonData:>"Ref:Luttrell1989c", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1990", ButtonData:>"Ref:Luttrell1990", ButtonStyle->"Hyperlink"], ")). Alternatively, if the state space of each stage of a 2-stage FMC is \ split into two (or more) lower dimensional subspaces, then the theory of \ self-supervision emerges (as in Luttrell (", ButtonBox["1991b", ButtonData:>"Ref:Luttrell1991b", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1992", ButtonData:>"Ref:Luttrell1992", ButtonStyle->"Hyperlink"], "))." }], "Text"], Cell[TextData[{ "The structure of this paper is as follows. In ", ButtonBox["section", ButtonData:>"Sect:BasicTheory", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:BasicTheory"], " the the idea of an FMC will be introduced, where Bayes' theorem is used \ to invert a Markov chain of probabilistic transformations. In ", ButtonBox["section", ButtonData:>"Sect:1and2StageFoldedMarkovChains", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:1and2StageFoldedMarkovChains"], " the relationship between FMCs and VQs and SOMs is derived, and it is \ shown how a 1-stage (or 2-stage) FMC contains a VQ (or SOM) as a special \ case. In ", ButtonBox["section", ButtonData:>"Sect:Coupled2StageFoldedMarkovChains", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:Coupled2StageFoldedMarkovChains"], " these results are extended to the case of a pair of coupled FMCs, and it \ is shown how self-supervision emerges naturally. In the first ", ButtonBox["appendix", ButtonData:>"Sect:AppendixContinuumDiscrete", ButtonStyle->"Hyperlink"], " the relationship between the continuum notation that is used in this \ paper and the more usual discrete notation is explained. In the second ", ButtonBox["appendix", ButtonData:>"Sect:AppendixOptimise", ButtonStyle->"Hyperlink"], " a more complete and technically rigorous derivation of the results of \ section 3 is presented. The main new results are contained in ", ButtonBox["section", ButtonData:>"Sect:1and2StageFoldedMarkovChains", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:1and2StageFoldedMarkovChains"], " and ", ButtonBox["section", ButtonData:>"Sect:Coupled2StageFoldedMarkovChains", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:Coupled2StageFoldedMarkovChains"], "." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Basic Theory" }], "Section", CellTags->"Sect:BasicTheory"], Cell[TextData[{ "The notation that is used for probabilities is very carefully chosen so as \ to avoid ambiguities that could arise if the notation ", Cell[BoxData[ \(TraditionalForm\`P(\[CenterEllipsis])\)]], " were used blindly to denote \"the probability density of ...\". However, \ occasional use of the ambiguous ", Cell[BoxData[ \(TraditionalForm\`P(\[CenterEllipsis])\)]], " notation is made where there is no possibility of an ambiguity arising. \ Also, a careful distinction is drawn between the notation that is used for \ inverse probabilities (obtained from Bayes' theorem), and the notation that \ is used for forward probabilities; the former always appear with a tilde over \ the ", Cell[BoxData[ \(TraditionalForm\`P\)]], ", which thus appears as ", Cell[BoxData[ \(TraditionalForm\`\(P\&~\)\)]], ". Also the state space(s) will be assumed to be continuous, except where \ otherwise noted. This is because the corresponding discrete space results can \ be written down by inspection of the continuum results, and because the \ meaning of a continuum calculation is usually more transparent than its \ discrete counterpart." }], "Text"], Cell[TextData[{ "The type of Markov chain that will be considered is shown in ", ButtonBox["figure", ButtonData:>"Fig:FoldedMarkovChain", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FoldedMarkovChain"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig2.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:FoldedMarkovChain"], Cell[TextData[{ "A folded Markov chain (i.e. both the forwards and backwards directions are \ represented). The top (bottom) half of the diagram represents the forward \ (backward) pass through the chain. The conditional probability ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(k + 1, k\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], \(k + 1\)], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"]}], ")"}]}], TraditionalForm]]], " is used as a probabilistic transformation that generates the probable \ states of layer ", Cell[BoxData[ \(TraditionalForm\`k + 1\)]], " from the state of layer ", Cell[BoxData[ \(TraditionalForm\`k\)]], ", and the conditional probability ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(k, k + 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "k"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], \(k + 1\)]}], ")"}], TraditionalForm]]], " is similarly used to generate the probable states of layer ", Cell[BoxData[ \(TraditionalForm\`k\)]], " from the state of layer ", Cell[BoxData[ \(TraditionalForm\`k + 1\)]], "; this is also referred to generically as stage ", Cell[BoxData[ \(TraditionalForm\`k\)]], " of the FMC. These two conditional probabilities are related by Bayes' \ theorem." }], "Caption"], Cell[TextData[{ "This is a folded Markov chain (FMC) which performs an ", Cell[BoxData[ \(TraditionalForm\`L\)]], "-stage transformation of an input vector ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " to an output vector ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "L"], TraditionalForm]]], " (via ", Cell[BoxData[ \(TraditionalForm\`L - 1\)]], " intermediate vectors ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], \(L - 1\)]}], TraditionalForm]]], "), and then performs the Bayes' inverse transformation to arrive \ eventually at a reconstructed input vector ", Cell[BoxData[ FormBox[ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], TraditionalForm]]], " . The delta function ", Cell[BoxData[ FormBox[ RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "L"], "-", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "L"]}], ")"}], TraditionalForm]]], " is used to ensure that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "L"], "=", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "L"]}], TraditionalForm]]], ". The conditional probabilities are related by Bayes' theorem as follows" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{\(\(P\&~\)\_\(k, k + 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], \(k + 1\)]}], ")"}], RowBox[{\(P\_\(k + 1\)\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], \(k + 1\)], ")"}]}], "=", RowBox[{ RowBox[{\(P\_\(k + 1, k\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], \(k + 1\)], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"]}], ")"}], RowBox[{\(P\_k\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"], ")"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:BayesTheorem"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{\(P\_k\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"], ")"}]}], TraditionalForm]]], " denotes the marginal probability (density) of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"], TraditionalForm]]], ". In order to construct any joint probability in the FMC system it is both \ necessary and sufficient to specify ", Cell[BoxData[ FormBox[ RowBox[{\(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], " and the ", Cell[BoxData[ \(TraditionalForm\`L\)]], " transformations ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], ",", RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], ",", "\[CenterEllipsis]", ",", RowBox[{\(P\_\(L, L - 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "L"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], \(L - 1\)]}], ")"}]}], TraditionalForm]]], ". Thus" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"P", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "L"], ";", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "L"]}], ",", "\[CenterEllipsis]", ",", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], ",", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}], ")"}], "=", RowBox[{ RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(P\_\(L, L - 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "L"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], \(L - 1\)]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "L"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "L"]}], ")"}], RowBox[{\(\(P\&~\)\_\(L - 1, L\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], \(L - 1\)], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "L"]}], ")"}], "\[CenterEllipsis]", RowBox[{\(\(P\&~\)\_\(1, 2\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:ProbabilityFMC"], Cell[TextData[{ "The marginal probability ", Cell[BoxData[ FormBox[ RowBox[{\(P\_k\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"], ")"}]}], TraditionalForm]]], " is obtained by integrating over all variables other than ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"], TraditionalForm]]], ", which yields" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_k\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"], ")"}]}], "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], "\[CenterEllipsis]", " ", SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], \(k - 1\)], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(P\_\(k, k - 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], \(k - 1\)]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Marginal"], Cell[TextData[{ "Note that Bayes' theorem guarantees that the marginal probabilities of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "k"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "k"], TraditionalForm]]], " are the same. In this paper our attention will be restricted to FMCs with \ 1 or 2 stages only (i.e. ", Cell[BoxData[ \(TraditionalForm\`L = 1\ or\ 2\)]], "). Furthermore, whenever an FMC is to be optimised, only its first stage \ will be optimised." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " 1- and 2-Stage Folded Markov Chains" }], "Section", CellTags->"Sect:1and2StageFoldedMarkovChains"], Cell["\<\ FMCs are especially interesting in adaptive network design theory, because \ they turn out to contain some well-known systems as special cases: a VQ is a \ special case of a 1-stage FMC, and an SOM is a special case of a 2-stage FMC. \ The success of these derivations relies on the similarities that exist \ between the following two situations:\ \>", "Text"], Cell[TextData[{ "1. Direct/inverse probabilities occur in Bayes' theorem as applied to a \ Markov chain, which might be used in the analysis of scattering problems in \ layered media, for instance. The sequence of processing operations is ", Cell[BoxData[ \(TraditionalForm\`source\[LongRightArrow]scattering\[LongRightArrow] inverse\ scattering\[LongRightArrow]reconstruction\)]], "." }], "Text"], Cell[TextData[{ "2. Encoding/decoding operations occur in VQs and SOMs, which might be used \ in the analysis of information transmission down a noisy communication \ channel, for instance. The sequence of processing operations is ", Cell[BoxData[ \(TraditionalForm\`source\[LongRightArrow]encode\[LongRightArrow] decode\[LongRightArrow]reconstruction\)]], "." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " 1-Stage Folded Markov Chain: Vector Quantiser" }], "Subsection"], Cell[TextData[{ "A 1-stage FMC is shown in ", ButtonBox["figure", ButtonData:>"Fig:FMC1Stage", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC1Stage"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig3.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:FMC1Stage"], Cell[TextData[{ "A 1-stage folded Markov chain. When ", Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], "=", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}]}], TraditionalForm]]], ", and ", Cell[BoxData[ \(TraditionalForm\`D\)]], " is minimised with respect to ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], ", this reduces to a vector quantiser with an infinite number of reference \ vectors (i.e. continuum limit). See the ", ButtonBox["appendix", ButtonData:>"Sect:AppendixContinuumDiscrete", ButtonStyle->"Hyperlink"], " for a detailed discussion of the relationship between the continuum and \ discrete cases. Bayes' theorem ensures that ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}], TraditionalForm]]], " cannot be varied independently of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], "." }], "Caption"], Cell[TextData[{ "The derivation starts from the definition of the average Euclidean error \ ", Cell[BoxData[ \(TraditionalForm\`D\)]], " between ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], TraditionalForm]]], " in a 1-stage FMC." }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "1"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage0"], Cell[TextData[{ "After integrating out the (irrelevant) ", Cell[BoxData[ FormBox[ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], TraditionalForm]]], " using the delta function ", Cell[BoxData[ FormBox[ RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], ", using Bayes' theorem in the form ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "=", RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}]}], TraditionalForm]]], " (i.e. ", ButtonBox["equation", ButtonData:>"Eq:BayesTheorem", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BayesTheorem"], " with ", Cell[BoxData[ \(TraditionalForm\`k = 0\)]], "), and rearranging the order of the integrations, this reduces to" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], \(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage1"], Cell[TextData[{ "The next step is to expand the norm as ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "\[RightDoubleBracketingBar]"}], "2"], "+", SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "\[RightDoubleBracketingBar]"}], "2"], "-", RowBox[{"2", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ".", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}]}]}], TraditionalForm]]], " and to perform the integrations where possible to obtain" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], " ", "[", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "\[RightDoubleBracketingBar]"}], "2"]}]}], "-", SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}]}], "\[RightDoubleBracketingBar]"}], "2"]}], "]"}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage1a"], Cell["which may be rewritten as", "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}]}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage2"], Cell[TextData[{ "The derivation of ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage2"], " from ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage1"], " is well-known; it says that the average Euclidean error between pairs of \ vectors drawn independently from ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}]}], TraditionalForm]]], " is twice the variance of vectors drawn from ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}]}], TraditionalForm]]], ". Finally, Bayes' theorem (i.e. ", ButtonBox["equation", ButtonData:>"Eq:BayesTheorem", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BayesTheorem"], " with ", Cell[BoxData[ \(TraditionalForm\`k = 0\)]], ") can be used to obtain the required result." }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], \(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}]}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage3"], Cell[TextData[{ ButtonBox["Equation", ButtonData:>"Eq:FMC1Stage3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage3"], " has all of the right structure to relate FMCs to VQs. It has a source of \ input vectors ", Cell[BoxData[ FormBox[ RowBox[{\(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], ", a \"soft\" encoder ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], ", and a reference vector ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}]}], TraditionalForm]]], " attached to each ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " with which to compare the input vector ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " to compute a Euclidean distortion. The only differences between this FMC \ and a standard VQ are:" }], "Text"], Cell[TextData[{ "1. ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " is not a winner-take-all encoder. Each input vector ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " is transformed into each possible output vector ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " with probability ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], ". In the language of neural networks, it is as if each possible output \ vector had an \"activity\" specified by ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], ". A winner-take-all would result if ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " were replaced by a probability whose mass was concentrated all at one \ point; this would be a delta function ", Cell[BoxData[ FormBox[ RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}], TraditionalForm]]], "." }], "Text"], Cell[TextData[{ "2. The reference vector ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}]}], TraditionalForm]]], " is dependent on the encoder ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], "; they are related by Bayes' theorem (i.e. ", ButtonBox["equation", ButtonData:>"Eq:BayesTheorem", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BayesTheorem"], " with ", Cell[BoxData[ \(TraditionalForm\`k = 0\)]], "). In a VQ the reference vector and the encoder are also related because \ the encoder is usually a nearest neighbour prescription, which in turn \ depends on the location of the reference vectors. It is not at all obvious \ that these two pictures (the FMC and the VQ) are related in a simple way." }], "Text"], Cell[TextData[{ "Now consider a modified form of ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage3"], " in which ", Cell[BoxData[ \(TraditionalForm\`D\)]], " becomes" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], \(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage4"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}]}], TraditionalForm]]], " has been replaced by the function ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], ". Functionally differentiate this expression for ", Cell[BoxData[ \(TraditionalForm\`D\)]], " with respect to ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " to obtain" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]], "=", RowBox[{\(-4\), RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], \(P\_\(1, 0\)\), RowBox[{ RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], " ", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "]"}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "The stationary point ", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold"]}]], "0"], "\[Prime]"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]], "=", "0"}], TraditionalForm]]], " is obtained when ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " satisfies" }], "Text", CellTags->"Ed:Change1"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "=", FractionBox[ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}]}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}]}]]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:CentroidFMC"], Cell[TextData[{ "By using Bayes' theorem (i.e. ", ButtonBox["equation", ButtonData:>"Eq:BayesTheorem", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BayesTheorem"], " with ", Cell[BoxData[ \(TraditionalForm\`k = 0\)]], ") this stationarity condition reduces to" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:CentroidFMC2"], Cell[TextData[{ "So the modified expression for ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage4"], " reduces to the original expression for ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage3"], " provided that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " is chosen to minimise ", Cell[BoxData[ \(TraditionalForm\`D\)]], ". This is a major simplification, because the coupling (via Bayes' \ theorem) between ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}]}], TraditionalForm]]], " that appeared in ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage3"], " can now safely be ignored by the simple trick of using ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage4"], " instead (with the proviso that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " should always be optimised so as to minimise ", Cell[BoxData[ \(TraditionalForm\`D\)]], ")." }], "Text"], Cell[TextData[{ "Make the following replacement in ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage4"] }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "\[LongRightArrow]", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:WinnerTakeAll"], Cell[TextData[{ "which converts ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " into a winner-take-all encoder. A winner-take-all encoder might appear \ intuitively to be the obvious solution to the problem of minimising ", Cell[BoxData[ \(TraditionalForm\`D\)]], " with respect to ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], ". However, other solutions may also be possible in general. A detailed \ derivation is given in the ", ButtonBox["appendix", ButtonData:>"Sect:AppendixOptimise", ButtonStyle->"Hyperlink"], " to show how this result emerges in the case of a Euclidean error \ function. Also a couple of simple counterexamples are presented in the ", ButtonBox["appendix", ButtonData:>"Para:CounterExamples", ButtonStyle->"Hyperlink"], " to show how non-winner-take-all encoders can also be valid solutions." }], "Text"], Cell[TextData[{ "With these replacements ", Cell[BoxData[ \(TraditionalForm\`D\)]], " becomes" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:ObjectiveVQ"], Cell[TextData[{ "which is exactly what would be written for the continuum version of a VQ \ (apart from the trivial overall factor of 2). The network representation of a \ VQ is shown in ", ButtonBox["figure", ButtonData:>"Fig:NetworkVQ", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:NetworkVQ"], " for a discrete-valued output, which should be compared with the FMC \ representation shown in ", ButtonBox["figure", ButtonData:>"Fig:FMC1Stage", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC1Stage"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig4.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:NetworkVQ"], Cell[TextData[{ "A discrete VQ represented as a network. The bottom layer is the input \ space (or ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], "), the top layer is the output space (or ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], "), and the connections between the two represent a soft encoding operation \ ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], ", akin to that used in ", ButtonBox["Yair ", ButtonData:>"Ref:YairZegerGersho1992", ButtonStyle->"Hyperlink"], StyleBox[ButtonBox["et al.", ButtonData:>"Ref:YairZegerGersho1992", ButtonStyle->"Hyperlink"], FontSlant->"Italic"], ButtonBox[" (1992)", ButtonData:>"Ref:YairZegerGersho1992", ButtonStyle->"Hyperlink"], ". A winner-take-all VQ uses an encoder of the form ", Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], "=", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}]}], TraditionalForm]]], ". The input and output layers of this network are represented in different \ ways: the input layer is the vector ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " represented as in ", ButtonBox["figure", ButtonData:>"Fig:FMC1Stage", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC1Stage"], ", whereas each node of the the output layer corresponds to exactly one \ possible state of the vector ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], ". Note also that the connections are not to be interpreted as weights in \ the conventional sense, rather they merely indicate the functional \ interdependence of the various parts of the network. An example of a winner \ in the output layer is represented by the open circle." }], "Caption"], Cell[TextData[{ "The gradient of ", Cell[BoxData[ \(TraditionalForm\`D\)]], " w.r.t. ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " is given by the functional derivative" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]], "=", RowBox[{\(-4\), RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{ RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}], " ", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "]"}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:ObjectiveVQDerivative"], Cell[TextData[{ "By inspecting the dependence of ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveVQ", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveVQ"], " on ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], ", and by setting ", Cell[BoxData[ FormBox[ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold"]}]], "0"], "\[Prime]"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveVQDerivative", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveVQDerivative"], " to zero, the following batch training prescription for minimising ", Cell[BoxData[ \(TraditionalForm\`D\)]], " can be obtained" }], "Text", CellTags->"Ed:Change2"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], "\[AlignmentMarker]", "=", RowBox[{GridBox[{ {\(arg\ min\)}, { SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]} }], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "\[AlignmentMarker]", "=", FractionBox[ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}], SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}]}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}]}]]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation", CellTags->"Eq:BatchTrainVQ"], Cell["These results merit the following remarks:", "Text"], Cell[TextData[{ "1. The function ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " can be interpreted as the continuum version of a VQ codebook, where ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " is the (continuum) code index and ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " is the code vector (or reference vector) associated with that index." }], "Text"], Cell[TextData[{ "2. The result for ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " corresponds to ", ButtonBox["equation", ButtonData:>"Eq:CentroidFMC", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:CentroidFMC"], " with ", Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "\[LongRightArrow]", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}]}], TraditionalForm]]], ", and it is the \"centroiding\" prescription for updating the code vectors \ after a batch of training data has been presented to a VQ, as used in the LBG \ algorithm (", ButtonBox["Linde ", ButtonData:>"Ref:LindeBuzoGray1980", ButtonStyle->"Hyperlink"], StyleBox[ButtonBox["et al.", ButtonData:>"Ref:LindeBuzoGray1980", ButtonStyle->"Hyperlink"], FontSlant->"Italic"], ButtonBox[" 1980", ButtonData:>"Ref:LindeBuzoGray1980", ButtonStyle->"Hyperlink"], ")." }], "Text"], Cell[TextData[{ "3. The result for ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], " is the \"nearest neighbour\" encoding prescription for encoding the input \ of a VQ." }], "Text"], Cell[TextData[{ "An on-line training prescription can also be obtained to implement updates \ to ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " after each input vector ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " is selected at random from ", Cell[BoxData[ FormBox[ RowBox[{\(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], ". The on-line prescription is" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "\[LongRightArrow]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "+", RowBox[{"\[Epsilon]", " ", RowBox[{ RowBox[{"\[Delta]", "[", RowBox[{\(x\_1\), "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}], " ", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "]"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:OnLineTrainVQ"], Cell[TextData[{ "Note that the delta function permits non-zero updates only for ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "=", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], TraditionalForm]]], "; this is the continuum version of updating the nearest neighbour code \ vector towards the input vector. The relationship between this continuum \ result and the corresponding discrete result is discussed in the ", ButtonBox["appendix", ButtonData:>"Sect:AppendixContinuumDiscrete", ButtonStyle->"Hyperlink"], "." }], "Text"], Cell["\<\ This completes the demonstration that an optimal 1-stage FMC is a VQ. Note \ that the use of a Euclidean error is sufficient (but not necessary) for this \ result to emerge. There are other choices of error function for which this \ result does not emerge.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " 2-Stage Folded Markov Chain: Self-Organising Map" }], "Subsection"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig5.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:FMC2Stage"], Cell[TextData[{ "A 2-stage folded Markov chain. This is basically the same as ", ButtonBox["figure", ButtonData:>"Fig:FMC1Stage", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC1Stage"], ", except that ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}]}], TraditionalForm]]], " causes the information that flows through the folded Markov chain to be \ further corrupted before it can begin its return journey." }], "Caption"], Cell["\<\ Because the derivation is so similar to the case of a 1-stage FMC, only an \ abbreviated derivation will be given. For simplicity, the following notation \ will be used\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(2, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "\[AlignmentMarker]", "=", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "\n", RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}]}], "\[AlignmentMarker]", "=", RowBox[{ RowBox[{\(P\_\(2, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation"], Cell[TextData[{ Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " is the probability (density) that ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], " will be generated by ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], ", taking into account all of the possible values that the intermediate \ state ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " might take. ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}], TraditionalForm]]], " is the corresponding inverse probability that is obtained from Bayes' \ theorem." }], "Text"], Cell[TextData[{ "Using this notation the expression for ", Cell[BoxData[ \(TraditionalForm\`D\)]], " becomes (compare ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage0", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage0"], ")" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "2"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "1"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "2"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], RowBox[{\(\(P\&~\)\_\(1, 2\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], \(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC2Stage0"], Cell[TextData[{ "which simplifies to (compare ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage3"], ")" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "2"], \(P\_\(2, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}]}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC2Stage3"], Cell[TextData[{ "Consider a modified form of ", Cell[BoxData[ \(TraditionalForm\`D\)]], " (compare ", ButtonBox["equation", ButtonData:>"Eq:FMC1Stage4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC1Stage4"], ")" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "2"], \(P\_\(2, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC2Stage4"], Cell[TextData[{ "Set ", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold"]}]], "0"], "\[Prime]"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]], "=", "0"}], TraditionalForm]]], " to obtain (compare ", ButtonBox["equation", ButtonData:>"Eq:CentroidFMC", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:CentroidFMC"], " and ", ButtonBox["equation", ButtonData:>"Eq:CentroidFMC2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:CentroidFMC2"], ")" }], "Text", CellTags->"Ed:Change3"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "=", RowBox[{ FractionBox[ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(2, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}]}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(2, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}]}]], "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "The modified expression for ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:FMC2Stage4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC2Stage4"], " reduces to the original expression in ", ButtonBox["equation", ButtonData:>"Eq:FMC2Stage3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC2Stage3"], " provided that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], TraditionalForm]]], " is chosen to minimise ", Cell[BoxData[ \(TraditionalForm\`\(\(D\)\(\ \)\)\)]], ", so ", ButtonBox["equation", ButtonData:>"Eq:FMC2Stage4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC2Stage4"], " will be used in preference to ", ButtonBox["equation", ButtonData:>"Eq:FMC2Stage3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC2Stage3"], " (with the proviso that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], TraditionalForm]]], " should always be chosen to minimise ", Cell[BoxData[ \(TraditionalForm\`D\)]], ")." }], "Text"], Cell[TextData[{ "Make the replacement ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "\[LongRightArrow]", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], " which converts ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " into a winner-take-all encoder (see the discussion following ", ButtonBox["equation", ButtonData:>"Eq:WinnerTakeAll", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:WinnerTakeAll"], ", and the ", ButtonBox["appendix", ButtonData:>"Sect:AppendixOptimise", ButtonStyle->"Hyperlink"], " for more details), and note that ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(2, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "\[AlignmentMarker]", "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}]}]}], TraditionalForm]]], ", to obtain ", Cell[BoxData[ \(TraditionalForm\`D\)]], " as (compare ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveVQ", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveVQ"], ")" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "2"], RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "The functional derivative ", Cell[BoxData[ FormBox[ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold"]}]], "0"], "\[Prime]"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]], TraditionalForm]]], " is thus (compare ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveVQDerivative", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveVQDerivative"], ")" }], "Text", CellTags->"Ed:Change4"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]], "=", RowBox[{\(-4\), RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], \(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], ")"}], RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "which leads to the following batch training prescription for minimising ", Cell[BoxData[ \(TraditionalForm\`D\)]], " (compare ", ButtonBox["equation", ButtonData:>"Eq:BatchTrainVQ", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BatchTrainVQ"], ")" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], "\[AlignmentMarker]", "=", RowBox[{GridBox[{ {\(arg\ min\)}, { SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]} }], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "2"], \(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "\[AlignmentMarker]", "=", FractionBox[ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}], SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}]}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}]}]]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation"], Cell[TextData[{ "and the following on-line training prescription (compare ", ButtonBox["equation", ButtonData:>"Eq:OnLineTrainVQ", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:OnLineTrainVQ"], ")" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "\[LongRightArrow]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], "+", RowBox[{"\[Epsilon]", " ", RowBox[{ RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}], " ", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], "]"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "These results correspond to the results that were reported in Luttrell (", ButtonBox["1989a", ButtonData:>"Ref:Luttrell1989a", ButtonStyle->"Hyperlink"], ",", ButtonBox["c", ButtonData:>"Ref:Luttrell1989c", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1990", ButtonData:>"Ref:Luttrell1990", ButtonStyle->"Hyperlink"], "). They can be interpreted as the generalisation of the VQ results in the \ previous section to the case where the output of the VQ is corrupted by the \ action of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], " before Bayes' theorem is then used in an attempt to reconstruct the input \ vector." }], "Text"], Cell[TextData[{ "This winner-take-all version of a 2-stage FMC turns out to be an SOM whose \ network representation is shown in ", ButtonBox["figure", ButtonData:>"Fig:NetworkSOM", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:NetworkSOM"], " for a discrete-valued output, which should be compared with the FMC \ representation that is shown in ", ButtonBox["figure", ButtonData:>"Fig:FMC2Stage", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC2Stage"] }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig6.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:NetworkSOM"], Cell[TextData[{ "A discrete SOM represented as a network. This is the same as the VQ \ network in ", ButtonBox["figure", ButtonData:>"Fig:NetworkVQ", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:NetworkVQ"], " with an additional stage of processing applied to its output layer. Each \ node in the hidden and output layer of this SOM network corresponds to \ exactly one possible state of the vector ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], ", respectively. ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], " serves as the SOM neighbourhood function by connecting together states of \ ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " so that they become ordered (across the page in this case). Note that the \ VQ network in ", ButtonBox["figure", ButtonData:>"Fig:NetworkVQ", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:NetworkVQ"], " does not have this ordering property, although the states of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " (or nodes) are still drawn in an ordered fashion, for convenience. An \ example of a winner in the hidden layer is drawn as an open circle, as are \ each of the corresponding soft winners in the output layer. An example of the \ degree to which each node in the output layer is activated is indicated by a \ histogram, which records ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], " for each possible state that ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], " might take." }], "Caption"], Cell["\<\ The SOM interpretation of these results for optimising a 2-stage FMC is as \ follows:\ \>", "Text"], Cell[TextData[{ "1. The function ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], TraditionalForm]]], " can be interpreted as the continuum version of the SOM reference vectors, \ where ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], " is the (continuum) index and ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], TraditionalForm]]], " is the reference vector associated with that index. The batch update \ prescription for ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], TraditionalForm]]], " is a generalisation of the LBG \"centroiding\" prescription (", ButtonBox["Linde 1980", ButtonData:>"Ref:LindeBuzoGray1980", ButtonStyle->"Hyperlink"], ") which accounts for the effect of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], "." }], "Text"], Cell[TextData[{ "2. The result for ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], TraditionalForm]]], " is not a \"nearest neighbour\" encoding prescription. Rather, it says \ that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], TraditionalForm]]], " is the value that ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " must take in order to ensure that the distortion ", Cell[BoxData[ \(TraditionalForm\`D\)]], " is minimised after taking into account the effect of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], ". Thus the nearest neighbour encoding prescription has become a minimum \ distortion encoding prescription. This reduces to the nearest neighbour \ encoding prescription when ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], "\[LongRightArrow]", RowBox[{"\[Delta]", "(", FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], "TraditionalForm"], ")"}]}], TraditionalForm]]], ", as expected." }], "Text"], Cell[TextData[{ "3. The on-line training prescription is the continuum version of the \ standard SOM training prescription, where ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], " plays the role of the SOM neighbourhood function. ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], " also has this interpretation in the batch training prescription." }], "Text"], Cell[TextData[{ "This completes the demonstration that an optimal 2-stage FMC is an SOM. \ Note that minimum distortion encoding is used, rather than nearest neighbour \ encoding, so this type of SOM is only an approximation to the standard SOM \ that was discussed in ", ButtonBox["Kohonen (1984)", ButtonData:>"Ref:Kohonen1984", ButtonStyle->"Hyperlink"], "." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Coupled 2-Stage Folded Markov Chains" }], "Section", CellTags->"Sect:Coupled2StageFoldedMarkovChains"], Cell[TextData[{ "In Luttrell (", ButtonBox["1991b", ButtonData:>"Ref:Luttrell1991b", ButtonStyle->"Hyperlink"], ", ", ButtonBox["1992", ButtonData:>"Ref:Luttrell1992", ButtonStyle->"Hyperlink"], ") some interesting results were reported where the behaviour of a \ multi-layer SOM could be interpreted as if the higher network layers were \ supervising the lower layers, and the term \"self-supervision\" was \ introduced to describe this effect. The purpose of this section is to show \ how these results can be derived from the theory of FMCs." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Splitting the Markov Chain State Spaces" }], "Subsection"], Cell[TextData[{ "The derivation starts by splitting each state of a 2-stage FMC into two \ lower dimensional pieces, as shown in ", ButtonBox["figure", ButtonData:>"Fig:FMC2StageSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC2StageSplit"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig7.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:FMC2StageSplit"], Cell[TextData[{ "A 2-stage folded Markov chain with each state split into two lower \ dimensional pieces. This is basically the same as ", ButtonBox["figure", ButtonData:>"Fig:FMC2Stage", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC2Stage"], ", except that the states have been exploded to reveal their internal \ structure." }], "Caption"], Cell[TextData[{ ButtonBox["Figure", ButtonData:>"Fig:FMC2StageSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC2StageSplit"], " can be obtained by making the following changes to the notation in ", ButtonBox["figure", ButtonData:>"Fig:FMC2Stage", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC2Stage"] }], "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "\[LongRightArrow]", RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], " ", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "\[LongRightArrow]", RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], ",", StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}]}]}, { RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "\[LongRightArrow]", RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], " ", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "\[LongRightArrow]", RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], ",", StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}]}]}, { RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "\[LongRightArrow]", RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], " ", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "\[LongRightArrow]", RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], ",", StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}]}]} }], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation", CellTags->"Eq:ChangeNotation"], Cell[TextData[{ "so the FMC in ", ButtonBox["figure", ButtonData:>"Fig:FMC2StageSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC2StageSplit"], " is the same as the FMC in ", ButtonBox["figure", ButtonData:>"Fig:FMC2Stage", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC2Stage"], ", apart from the notation that is used to describe its state spaces. This \ tautology is motivated by the need to break up high dimensional spaces, such \ as might occur in image processing problems, into a number of coupled lower \ dimensional spaces. For instance, if the coupling between the FMCs is weak, \ then a series expansion about zero coupling strength could be attempted. The \ lowest order term in this expansion would correspond to a pair of uncoupled \ FMCs, and the higher order terms would correspond to interactions between the \ FMCs." }], "Text"], Cell[TextData[{ "If the new notation in ", ButtonBox["equation", ButtonData:>"Eq:ChangeNotation", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ChangeNotation"], " is used together with the generalisation of ", ButtonBox["equation", ButtonData:>"Eq:WinnerTakeAll", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:WinnerTakeAll"] }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], "\[LongRightArrow]", RowBox[{"\[Delta]", "[", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}]}], RowBox[{"\[Delta]", "[", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:WinnerTakeAllSplit"], Cell[TextData[{ "in the expression for the distortion ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in a 2-stage FMC (see ", ButtonBox["equation", ButtonData:>"Eq:FMC2Stage3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:FMC2Stage3"], "), and noting that ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(2, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "\[AlignmentMarker]", "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}]}]}], TraditionalForm]]], ", then ", Cell[BoxData[ \(TraditionalForm\`D\)]], " reduces to" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(P\_0\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontSlant->"Italic"]}]], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], ",", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}], RowBox[{"(", RowBox[{ SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox["-", FontWeight->"Plain"], RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], \(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}], "\[RightDoubleBracketingBar]"}], "2"], "+", SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox["-", FontWeight->"Plain"], RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], \(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}], "\[RightDoubleBracketingBar]"}], "2"]}], ")"}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:ObjectiveSplit"], Cell[TextData[{ "where the Euclidean error has been split into a sum of contributions from \ the ", Cell[BoxData[ \(TraditionalForm\`a\)]], " and the ", Cell[BoxData[ \(TraditionalForm\`b\)]], " subspaces of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], ". The Euclidean error terms have an interesting structure. For instance, \ in FMC ", Cell[BoxData[ \(TraditionalForm\`a\)]], " the Euclidean distance between ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " is computed, which explicitly depends via ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " on the outputs ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " of both FMC ", Cell[BoxData[ \(TraditionalForm\`a\)]], " and FMC ", Cell[BoxData[ \(TraditionalForm\`b\)]], ". In the previous section terms like ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}]}], TraditionalForm]]], " turned out to correspond to SOM reference vectors, so by analogy it is \ expected that ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " (", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]], ") will also turn out to be the reference vectors for a pair of coupled \ SOMs corresponding to FMC ", Cell[BoxData[ \(TraditionalForm\`a\)]], " and FMC ", Cell[BoxData[ \(TraditionalForm\`b\)]], "." }], "Text"], Cell[TextData[{ "Unfortunately, because of the coupling between FMC ", Cell[BoxData[ \(TraditionalForm\`a\)]], " and FMC ", Cell[BoxData[ \(TraditionalForm\`\(\(b\)\(\ \)\)\)]], ", ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " (", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]], ") depend on both ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], TraditionalForm]]], ". The purpose of the following derivation is to find an approximate way of \ optimising ", Cell[BoxData[ \(TraditionalForm\`D\)]], " that does not involve these simultaneous dependencies on both ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], TraditionalForm]]], ", but uses quantities like ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " rather than ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " for ", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]], "." }], "Text"], Cell[TextData[{ "Thus each term can be split in the following manner for ", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]] }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], \(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}], "\[RightDoubleBracketingBar]"}], "2"], "=", SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], \(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}], ")"}], "+", RowBox[{"(", RowBox[{ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], \(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], "-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], \(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:EuclideanSplit"], Cell[TextData[{ "where the distinction between ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " has been carefully used to separate the ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " terms. The following two definitions are introduced for ", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]] }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{\(\(D\_0\) \((k)\)\), "\[Congruent]", "\[AlignmentMarker]", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], \(P\_0\), RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], ",", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], \(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{\(\(D\_1\) \((k)\)\), "\[Congruent]", "\[AlignmentMarker]", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontSlant->"Italic"]}]], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], \(P\_2\), RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], "-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], \(\(P\&~\)\_\(0, 2\)\), RowBox[{"(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation", CellTags->"Eq:ObjectiveSplitDefineNotation"], Cell["and used together with Bayes' theorem in the form", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{\(P\_0\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], ",", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}]}], "=", RowBox[{ RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], RowBox[{\(P\_2\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:BayesTheoremSplit"], Cell[TextData[{ "to obtain ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in the form" }], "Text"], Cell[BoxData[ \(TraditionalForm\`D = \(D\_0\)(a) + \(D\_0\)(b) - \(D\_1\)(a) - \(D\_1\)( b)\)], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:ObjectiveSplitNotation"], Cell[TextData[{ "The first two terms in ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplitNotation", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplitNotation"], " have a structure that is simpler than in ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplit"], ". For instance ", Cell[BoxData[ \(TraditionalForm\`\(D\_0\)(a)\)]], " depends on the Euclidean distance between ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], ", which depends only on the output of FMC ", Cell[BoxData[ \(TraditionalForm\`a\)]], " and not on the output of FMC ", Cell[BoxData[ \(TraditionalForm\`b\)]], ". An analogous remark applies to ", Cell[BoxData[ \(TraditionalForm\`\(D\_0\)(b)\)]], "." }], "Text"], Cell[TextData[{ "The last two terms in ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplitNotation", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplitNotation"], " contain the undesirable terms such as ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], StyleBox[")", FontWeight->"Bold"]}], TraditionalForm]]], ". In the following derivation these terms will be discarded to obtain an \ approximate scheme for minimising ", Cell[BoxData[ \(TraditionalForm\`D\)]], "." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Least Upper Bound Optimisation Scheme" }], "Subsection"], Cell["Note that the following inequalities hold", "Text"], Cell[BoxData[ FormBox[GridBox[{ {\(\(D\_0\)(a) \[GreaterEqual] 0\), " ", \(\(D\_0\)(b) \[GreaterEqual] 0\), " ", \(\(D\_1\)(a) \[GreaterEqual] 0\), " ", \(\(D\_1\)(b) \[GreaterEqual] 0\), " ", \(D \[GreaterEqual] 0\)} }], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "These lead to the following constraint on ", Cell[BoxData[ \(TraditionalForm\`D\)]] }], "Text"], Cell[BoxData[ \(TraditionalForm\`0 \[LessEqual] D \[LessEqual] \(D\_0\)(a) + \(D\_0\)(b)\)], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:ObjectiveSplitUpperBound"], Cell[TextData[{ "Although ideally ", Cell[BoxData[ \(TraditionalForm\`D\)]], " itself should be minimised, it turns out to be much simpler to minimise \ its upper bound ", Cell[BoxData[ \(TraditionalForm\`\(D\_0\)(a) + \(D\_0\)(b)\)]], ". This least upper bound prescription achieves what is required, namely \ the elimination of the undesirable ", Cell[BoxData[ \(TraditionalForm\`\(D\_1\)(a)\)]], " and ", Cell[BoxData[ \(TraditionalForm\`\(D\_1\)(b)\)]], " terms which depend on ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " for ", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]], ". This approximate prescription becomes exact in the limit where the \ coupling between the FMCs in ", ButtonBox["figure", ButtonData:>"Fig:FMC2StageSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC2StageSplit"], " tends to zero." }], "Text"], Cell[TextData[{ "This approximate approach to optimising the FMC is shown in ", ButtonBox["figure", ButtonData:>"Fig:FMC2StageSplitApprox", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:FMC2StageSplitApprox"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig8.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:FMC2StageSplitApprox"], Cell[TextData[{ "An approximation to a 2-stage folded Markov chain with each state split \ into two lower dimensional pieces. Only the ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " part is indicated on the reverse part of the chain. The approximation \ arises because ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], ", rather then ", Cell[BoxData[ FormBox[ RowBox[{"(", StyleBox[ RowBox[{ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], ",", SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], FontWeight->"Bold"], ")"}], TraditionalForm]]], ", is used as the initial state on the reverse part of the chain. An \ analogous discussion holds for ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], "." }], "Caption"], Cell[TextData[{ "At this point it is appropriate to introduce a modified form of ", Cell[BoxData[ \(TraditionalForm\`\(D\_0\)(k)\)]], " in which ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], StyleBox[ SubsuperscriptBox[ StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplitDefineNotation", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplitDefineNotation"], " is replaced by the function ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], TraditionalForm]]], " for ", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]] }], "Text"], Cell[BoxData[ FormBox[ RowBox[{\(\(D\_0\)(k)\), "=", "\[AlignmentMarker]", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], \(P\_0\), RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], ",", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:ObjectiveSplitDefineNotation2"], Cell[TextData[{ "The functional derivative ", Cell[BoxData[ FormBox[ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]], TraditionalForm]]], " then becomes for ", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]] }], "Text", CellTags->"Ed:Change5"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ FractionBox[ RowBox[{ StyleBox[ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontSlant->"Plain"], StyleBox["D", FontSlant->"Italic"]}]], StyleBox["0", FontSlant->"Plain"]], FontSlant->"Italic"], "(", StyleBox["k", FontSlant->"Italic"], StyleBox[")", FontSlant->"Italic"]}], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]], "=", "\[AlignmentMarker]", RowBox[{\(-4\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], \(P\_0\), RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{ RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], ",", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}], " ", "[", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], "]"}]}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:ObjectiveSplitDerivative"], Cell[TextData[{ "and the cross derivatives ", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{ StyleBox[ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontSlant->"Plain"], StyleBox["D", FontSlant->"Italic"]}]], StyleBox["0", FontSlant->"Plain"]], FontSlant->"Italic"], "(", StyleBox["a", FontSlant->"Italic"], StyleBox[")", FontSlant->"Italic"]}], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{ StyleBox[ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontSlant->"Plain"], StyleBox["D", FontSlant->"Italic"]}]], StyleBox["0", FontSlant->"Plain"]], FontSlant->"Italic"], "(", StyleBox["b", FontSlant->"Italic"], StyleBox[")", FontSlant->"Italic"]}], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]], TraditionalForm]]], " are zero. The stationary point ", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["\[Delta]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]], "=", "0"}], TraditionalForm]]], " is obtained when ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], TraditionalForm]]], " satisfies for ", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]] }], "Text", CellTags->"Ed:Change6"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], "\[AlignmentMarker]", "=", FractionBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(P\_0\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], ",", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}], StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}], RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(P\_0\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], ",", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}]}]}]]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "Note that this result can readily be generalised to an arbitrary choice of \ ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " (i.e. not assuming ", ButtonBox["equation", ButtonData:>"Eq:WinnerTakeAllSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:WinnerTakeAllSplit"], ")." }], "Text"], Cell[TextData[{ "By using Bayes' theorem in the form shown in ", ButtonBox["equation", ButtonData:>"Eq:BayesTheoremSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BayesTheoremSplit"], " this reduces to" }], "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], "\[AlignmentMarker]", "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}], " ", \(k = a, b\)} }], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "Thus the modified expression for ", Cell[BoxData[ \(TraditionalForm\`\(D\_0\)(k)\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplitDefineNotation2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplitDefineNotation2"], " reduces to the original expression for ", Cell[BoxData[ \(TraditionalForm\`\(D\_0\)(k)\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplitDefineNotation", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplitDefineNotation"], " provided that ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], TraditionalForm]]], " is chosen to minimise ", Cell[BoxData[ \(TraditionalForm\`\(D\_0\)(k)\)]], ", so ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplitDefineNotation2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplitDefineNotation2"], " will be used in preference to ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplitDefineNotation", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplitDefineNotation"], " (with the proviso that ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], TraditionalForm]]], " should always be chosen to minimise ", Cell[BoxData[ \(TraditionalForm\`\(D\_0\)(k)\)]], ")." }], "Text"], Cell[TextData[{ "When the upper bound of ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplitUpperBound", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplitUpperBound"], " is minimised with respect to ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], TraditionalForm]]], " using ", ButtonBox["equation", ButtonData:>"Eq:ObjectiveSplitDefineNotation2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:ObjectiveSplitDefineNotation2"], " it yields for ", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]] }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], "=", RowBox[{GridBox[{ {\(arg\ min\)}, { StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"]} }], "[", RowBox[{ RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], \(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}], "+", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], \(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}], "]"}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:BatchTrainSplit"], Cell[TextData[{ "and the following on-line training prescription for ", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]] }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], "\[LongRightArrow]", RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], "+", RowBox[{"\[Epsilon]", " ", RowBox[{ RowBox[{\(P\_\(2, 1\)\), "[", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], ",", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}]}], "]"}], " ", "[", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontSlant->"Italic"], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "-", RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], "]"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:OnLineTrainSplit"], Cell[TextData[{ "These results correspond to the self-supervised training scheme that was \ proposed in Luttrell (", ButtonBox["1991b", ButtonData:>"Ref:Luttrell1991b", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1992", ButtonData:>"Ref:Luttrell1992", ButtonStyle->"Hyperlink"], "), which was the result of a detailed study was made of the problem of \ designing an encoder/decoder for a pair of communication channels whose \ transmitted information was degraded by a noise process (both external noise \ and noisy coupling between the channels). The improvement in performance when \ channel coupling is taken into account was shown to be significant, so the \ least upper bound approximation is justified in hindsight for this type of \ system." }], "Text"], Cell["The following remarks can be made about these results:", "Text"], Cell[TextData[{ "1. The functions ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox[ SuperscriptBox["x", StyleBox["\[Prime]", FontWeight->"Plain"]], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], TraditionalForm]]], " are the continuum versions of the reference vectors of a pair of SOMs \ corresponding to FMC ", Cell[BoxData[ \(TraditionalForm\`a\)]], " and FMC ", Cell[BoxData[ \(TraditionalForm\`b\)]], ", respectively." }], "Text"], Cell[TextData[{ "2. The results for ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " are modified forms of the minimum distortion encoding prescription, in \ which the coupling between the FMC ", Cell[BoxData[ \(TraditionalForm\`a\)]], " and FMC ", Cell[BoxData[ \(TraditionalForm\`b\)]], " manifests itself through ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], "." }], "Text"], Cell[TextData[{ "3. In both the batch and the on-line training prescription ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " play the role of neighbourhood functions for SOM ", Cell[BoxData[ \(TraditionalForm\`a\)]], " and SOM ", Cell[BoxData[ \(TraditionalForm\`b\)]], ", respectively. However, the coupling between the SOMs causes these \ neighbourhood functions to be dependent on the input data, as will be \ discussed in more detail below." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Network Representation of Self-Supervision" }], "Subsection"], Cell[TextData[{ "In ", ButtonBox["figure", ButtonData:>"Fig:NetworkSelfSupervision", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:NetworkSelfSupervision"], " the network representation of the pair of coupled SOMs is shown for a \ discrete-valued output." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig9.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:NetworkSelfSupervision"], Cell[TextData[{ "A pair of coupled discrete SOMs represented as a pair of coupled networks \ of the type shown in ", ButtonBox["figure", ButtonData:>"Fig:NetworkSOM", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:NetworkSOM"], ". As derived above, the coupling ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " between networks ", Cell[BoxData[ \(TraditionalForm\`a\)]], " and ", Cell[BoxData[ \(TraditionalForm\`b\)]], " expresses itself as a pair of data dependent neighbourhood functions ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], ". When the first stage of each of these coupled SOMs is optimised, taking \ account of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], ", the encoding functions ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " become mutually coupled." }], "Caption"], Cell[TextData[{ "The detailed interpretation of ", ButtonBox["equation", ButtonData:>"Eq:BatchTrainSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BatchTrainSplit"], " and ", ButtonBox["equation", ButtonData:>"Eq:OnLineTrainSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:OnLineTrainSplit"], " is shown in ", ButtonBox["figure", ButtonData:>"Fig:SelfSupervision", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:SelfSupervision"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/fig10.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:SelfSupervision"], Cell[TextData[{ "Diagram showing the detailed operation of a pair of coupled FMCs. For \ simplicity, ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " is assumed to sit in the same vector space as ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], ". Examples of the winners ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " are indicated by the open circles. These are then jointly smeared into \ the distribution ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], ", which is drawn as a 2-dimensional histogram. As derived above, the \ optimisation depends on a pair of neighbourhood functions ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], ", which are the marginal probabilities of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], ", and which are drawn as 1-dimensional histograms in the diagram. These \ data-dependent neighbourhood functions not only influence the optimisation of \ the FMC, but also determine the winners that should have been used in the \ first place." }], "Caption"], Cell[TextData[{ "The marginal probabilities ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " are obtained by projecting ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " onto ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], TraditionalForm]]], ", respectively. The data dependence of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " causes these marginal probabilities to be data dependent. In particular, \ they can be biassed as shown in ", ButtonBox["figure", ButtonData:>"Fig:SelfSupervision", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:SelfSupervision"], ", which causes the neighbourhood functions (in the SOM interpretation) in \ ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " space to be biassed." }], "Text"], Cell[TextData[{ "The data dependence has another subtle side effect in ", ButtonBox["figure", ButtonData:>"Fig:SelfSupervision", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:SelfSupervision"], ". The input transformations ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["0", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["0", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]]}], ")"}], TraditionalForm]]], " depend on the marginal probabilities ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], ", because the input transformations satisfy a minimum distortion criterion \ which depends on these marginal probabilities (see ", ButtonBox["equation", ButtonData:>"Eq:BatchTrainSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BatchTrainSplit"], "). However, the marginal probabilities themselves are data dependent, \ because they depend on ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], ", which in turn depend on the input transformations. Overall, the marginal \ probabilities and the input transformations are mutually dependent, which \ makes the minimum distortion encoding prescription quite subtle to implement \ in this case." }], "Text"], Cell[TextData[{ "Further details on self-supervision can be found in Luttrell (", ButtonBox["1991b", ButtonData:>"Ref:Luttrell1991b", ButtonStyle->"Hyperlink"], "; ", ButtonBox["1992", ButtonData:>"Ref:Luttrell1992", ButtonStyle->"Hyperlink"], "), where a detailed discussion and numerical simulation of the \ consequences of using a particular type of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[",", FontWeight->"Bold"], RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " are presented, a comparison is made between nearest neighbour and minimum \ distortion encoding, and a comparison is made between using mutually \ dependent neighbourhood functions ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["k", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " (", Cell[BoxData[ \(TraditionalForm\`k = a, b\)]], ") and independent neighbourhood functions ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["a", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], StyleBox["2", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"], "|", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["b", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], TraditionalForm]]], "." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Conclusions" }], "Section"], Cell["\<\ In this paper it has been demonstrated that VQ theory, SOM theory, and the \ theory of self-supervision all emerge naturally when an FMC is optimised so \ as to minimise the expected Euclidean error between an input vector and its \ attempted reconstruction (using Bayes' theorem).\ \>", "Text"], Cell["\<\ FMC theory can be used to facilitate many computations that would otherwise \ be theoretically and/or numerically intractable. The \"soft\" probabilities \ that are used in the FMC are easier to compute with than the \"hard\" delta \ functions in the corresponding winner-take-all VQs and SOMs. The results \ contained in this paper guarantee that these \"soft\" computations reduce to \ the required \"hard\" computations when the first stage of the FMC is \ optimised.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Appendix" }], "Section", CellTags->"Sect:Appendix"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", "Relationship between Continuum and Discrete Vector Quantisers" }], "Subsection", CellTags->"Sect:AppendixContinuumDiscrete"], Cell["\<\ Throughout this paper continuum notation is used. In the case of a VQ this \ has the effect that the index that is used to select the winning reference \ vector is assumed to be a continuous-valued quantity, rather than a \ discrete-valued quantity. The purpose of this appendix is to relate the \ continuum case to the discrete case.\ \>", "Text"], Cell[TextData[{ "It is sufficient to discuss the meaning of the on-line training \ prescription in ", ButtonBox["equation", ButtonData:>"Eq:BatchTrainVQ", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BatchTrainVQ"], " and ", ButtonBox["equation", ButtonData:>"Eq:OnLineTrainVQ", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:OnLineTrainVQ"], ", which is presented again here for convenience." }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], "\[AlignmentMarker]", "=", RowBox[{GridBox[{ {\(arg\ min\)}, { SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]} }], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "\[AlignmentMarker]", "\[LongRightArrow]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "+", RowBox[{"\[Epsilon]", " ", RowBox[{ RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}], " ", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "]"}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation", CellTags->"Eq:OnLineTrainVQ2"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " is assumed to lie in the unit ", Cell[BoxData[ \(TraditionalForm\`N\)]], "-dimensional hypercube. The discrete counterpart of this on-line \ prescription is" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], "\[AlignmentMarker]", "=", RowBox[{GridBox[{ {\(arg\ min\)}, { SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"]} }], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], RowBox[{"0", ",", SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"]}]]}], "\[RightDoubleBracketingBar]"}], "2"]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], RowBox[{"0", ",", SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"]}]], "\[AlignmentMarker]", "\[LongRightArrow]", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], RowBox[{"0", ",", SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"]}]]}], "+", RowBox[{"\[Epsilon]", " ", RowBox[{ SubscriptBox["\[Delta]", RowBox[{ SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"], ",", RowBox[{ SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}]], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], RowBox[{"0", ",", SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"]}]]}], ")"}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation", CellTags->"Eq:OnLineTrainVQ2Discrete"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"], TraditionalForm]]], " is a point on an ", Cell[BoxData[ \(TraditionalForm\`N\)]], "-dimensional cubic lattice." }], "Text"], Cell[TextData[{ "The relationship between the nearest neighbour encoding prescriptions ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"], RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], " is simple to understand, so no further comment is required. On the other \ hand, the the interpretation of the update prescription for ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " is quite subtle, so it will now be discussed in some detail." }], "Text"], Cell[TextData[{ "In order to interpret the continuum prescription correctly it is necessary \ to integrate over a small region ", Cell[BoxData[ \(TraditionalForm\`S\)]], " with volume ", Cell[BoxData[ FormBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["V", FontSlant->"Italic"]}]], TraditionalForm]]], " that encloses the point ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "=", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], ", and to assume that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " is a smooth function of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " (for a justification of this assumption, see the last paragraph of this \ ", ButtonBox["appendix", ButtonData:>"Para:Smooth", ButtonStyle->"Hyperlink"], "). This leads to the result" }], "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ RowBox[{ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "\[AlignmentMarker]", "\[LongRightArrow]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "+", RowBox[{ FractionBox["\[Epsilon]", StyleBox[ RowBox[{"\[Delta]", StyleBox["V", FontSlant->"Italic"]}]]], " ", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], "]"}]}], " ", RowBox[{ RowBox[{"where", " ", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], "\[Element]", RowBox[{"S", " ", "and", " ", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "\[Element]", "S"}]} }], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:OnLineTrainVQSmooth"], Cell[TextData[{ "which can be compared directly with the update prescription for ", Cell[BoxData[ FormBox[ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], RowBox[{"0", ",", SubscriptBox[ StyleBox["k", FontWeight->"Bold"], "1"]}]], TraditionalForm]]], ". Note that ", Cell[BoxData[ FormBox[ FractionBox["\[Epsilon]", StyleBox[ RowBox[{"\[Delta]", StyleBox["V", FontSlant->"Italic"]}]]], TraditionalForm]]], ", rather than ", Cell[BoxData[ \(TraditionalForm\`\[Epsilon]\)]], ", determines the size of the updates of the reference vectors that are \ attached to points ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " inside ", Cell[BoxData[ \(TraditionalForm\`S\)]], ". The smaller the volume ", Cell[BoxData[ FormBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["V", FontSlant->"Italic"]}]], TraditionalForm]]], " is, the smaller ", Cell[BoxData[ \(TraditionalForm\`\[Epsilon]\)]], " has to be to keep the updates the same size. The most natural \ prescription is to impose a lower bound ", Cell[BoxData[ FormBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["L", FontSlant->"Italic"]}]], TraditionalForm]]], " on the length scale of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " of interest, which then imposes a natural size on the volume of ", Cell[BoxData[ \(TraditionalForm\`S\)]], " such that ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["V", FontSlant->"Italic"]}]], StyleBox["=", FontSlant->"Italic"], SuperscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["L", FontSlant->"Italic"]}]], "N"]}], TraditionalForm]]], ". Since ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " is assumed to lie in the unit hypercube, then this prescription leads to \ a finite effective number of reference vectors given by ", Cell[BoxData[ FormBox[ SuperscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["L", FontSlant->"Italic"]}]], \(-N\)], TraditionalForm]]], ", which would correspond to an ", Cell[BoxData[ \(TraditionalForm\`N\)]], "-dimensional cubic lattice whose size was ", Cell[BoxData[ FormBox[ FractionBox["1", StyleBox[ RowBox[{"\[Delta]", StyleBox["L", FontSlant->"Italic"]}]]], TraditionalForm]]], " lattice spacings in each dimension." }], "Text", CellTags->"Ed:Change7"], Cell[TextData[{ "An interesting side effect of introducing this inner length scale ", Cell[BoxData[ FormBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["L", FontSlant->"Italic"]}]], TraditionalForm]]], " (which is effectively a regularisation constant) is that it automatically \ generates a SOM on-line training prescription. To see this, note that in ", ButtonBox["equation", ButtonData:>"Eq:OnLineTrainVQSmooth", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:OnLineTrainVQSmooth"], " a whole volume ", Cell[BoxData[ FormBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["V", FontSlant->"Italic"]}]], TraditionalForm]]], " of reference vectors ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " in the neighbourhood ", Cell[BoxData[ \(TraditionalForm\`S\)]], " of the point ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "=", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], " is updated. This is precisely the type of update prescription that \ creates a SOM. Furthermore, this justifies with hindsight the assumption made \ earlier that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], " is a smooth function of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], "." }], "Text", CellTags->"Para:Smooth"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Full Optimisation of 2-Stage Folded Markov Chains" }], "Subsection", CellTags->"Sect:AppendixOptimise"], Cell[TextData[{ "In this appendix it will be shown that if a 2-stage FMC with a Euclidean \ error function is optimised with respect to ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], TraditionalForm]]], ", then it reduces to ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "=", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], ". Because a 2-stage FMC contains a 1-stage FMC as a special case, it is \ necessary to present the derivation only for a 2-stage FMC. The result that \ is obtained depends on certain properties of the Euclidean error function; it \ is not a general property of FMCs." }], "Text"], Cell[TextData[{ "There are two constraints that must be respected during the optimisation \ of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], ". Total probability must be conserved" }], "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}]}], "=", "1"}], " ", RowBox[{"for", " ", "all", " ", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}]} }], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "and ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], " must vary in response to variations of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " in such a way that Bayes' theorem is respected (i.e ", ButtonBox["equation", ButtonData:>"Eq:BayesTheorem", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:BayesTheorem"], " with ", Cell[BoxData[ \(TraditionalForm\`k = 0\)]], ")" }], "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ RowBox[{\(\(P\&~\)\_\(0, 1\)\), RowBox[{ RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], "[", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], ")"}]}]}], "]"}]}], "=", RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}]}], " ", RowBox[{ RowBox[{"for", " ", "all", " ", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}]} }], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation"], Cell[TextData[{ "The first constraint will be imposed for all ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " by using a Lagrange multiplier function ", Cell[BoxData[ FormBox[ RowBox[{"\[Lambda]", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], TraditionalForm]]], ", whereas the second constraint will be imposed by using Bayes' theorem to \ eliminate ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}]}], TraditionalForm]]], " from ", Cell[BoxData[ \(TraditionalForm\`D\)]], "." }], "Text"], Cell[TextData[{ "The overall quantity ", Cell[BoxData[ \(TraditionalForm\`D\&_\)]], " to be minimised is therefore" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{\(D\&_\), "=", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "2"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "1"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "2"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], RowBox[{\(\(P\&~\)\_\(1, 2\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}], "-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{"\[Lambda]", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective2StageLagrange"], Cell[TextData[{ "Integration over ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], " and use of Bayes' theorem on both ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(1, 2\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{\(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}]}], TraditionalForm]]], " transforms ", Cell[BoxData[ \(TraditionalForm\`D\&_\)]], " into a form that is suitable for differentiation with respect to the \ sought after quantity ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]] }], "Text"], Cell[BoxData[ FormBox[ RowBox[{\(D\&_\), "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "1"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "2"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}], ")"}], \(P\_0\), RowBox[{"(", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], ")"}], FractionBox[ RowBox[{ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}]}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "\[Prime]"], "|", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], ")"}]}]}]], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}], "-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{"\[Lambda]", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective2StageLagrange2"], Cell[TextData[{ "Functional differentiation of ", Cell[BoxData[ \(TraditionalForm\`D\&_\)]], " with respect to ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"]}], ")"}], TraditionalForm]]], ", and using the symmetry of ", Cell[BoxData[ \(TraditionalForm\`D\&_\)]], " under interchange of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], TraditionalForm]]], ", then leads to" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ FractionBox[\(\[Delta] D\&_\), RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Italic"]}]], \(1, 0\)], "(", RowBox[{ SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"]}], ")"}]], "=", "\[AlignmentMarker]", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "1"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "2"], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], ")"}], FractionBox[ RowBox[{\(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], \(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}]}], RowBox[{"(", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], ")"}]}]}], ")"}]], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "1"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "2"], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], ")"}], RowBox[{ FractionBox[ RowBox[{\(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], \(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}]}], SuperscriptBox[ RowBox[{"[", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["u", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["u", FontWeight->"Bold"], "0"], ")"}]}]}], "]"}], "2"]], "[", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["v", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["v", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ StyleBox["v", FontWeight->"Bold"], "0"], "-", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ StyleBox["v", FontWeight->"Bold"], "1"], "-", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["v", FontWeight->"Bold"], "1"]}], ")"}], \(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["v", FontWeight->"Bold"], "0"], ")"}]}]}], "]"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], RowBox[{"\[Lambda]", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"]}], ")"}]}]}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective2StageLagrange3"], Cell["Using Bayes' theorem this simplifies to", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ FractionBox[\(\[Delta] D\&_\), RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Italic"]}]], \(1, 0\)], "(", RowBox[{ SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"]}], ")"}]], "=", "\[AlignmentMarker]", RowBox[{"2", \(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "2"], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{"\[Integral]", RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], "\[Prime]"], \(\(P\&~\)\_\(1, 2\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], \(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "-", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"-", RowBox[{\(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "2"], RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"]}], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1"], \(\(P\&~\)\_\(1, 2\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "1"], RowBox[{\(\(P\&~\)\_\(1, 2\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "0"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], \(\(P\&~\)\_\(0, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"-", RowBox[{"\[Lambda]", "(", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective2StageLagrange4"], Cell["Expansion and factorisation eventually lead to the result", "Text"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[\(\[Delta] D\&_\), RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Italic"]}]], \(1, 0\)], "(", RowBox[{ SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"]}], ")"}]], "=", RowBox[{ RowBox[{"2", RowBox[{\(P\_0\), "(", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "2"], \(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ SuperscriptBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "\[Prime]"], "|", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "1"], RowBox[{\(\(P\&~\)\_\(0, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"]}], ")"}], RowBox[{\(\(P\&~\)\_\(1, 2\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "1"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}]}], "-", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}], "-", RowBox[{"\[Lambda]", "(", SubscriptBox[ StyleBox["z", FontWeight->"Bold"], "0"], ")"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective2StageLagrange5"], Cell[TextData[{ "The stationarity condition ", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[\(\[Delta] D\&_\), RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Italic"]}]], \(1, 0\)], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]], "=", "0"}], TraditionalForm]]], " is therefore" }], "Text", CellTags->"Ed:Change8"], Cell[BoxData[ FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{ RowBox[{"2", \(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "2"], \(P\_\(2, 1\)\), RowBox[{"(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}]}], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}], "=", RowBox[{"\[Lambda]", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}]}], TraditionalForm]], "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Fig:FMC2StageStationarity"], Cell[TextData[{ "The right hand side of ", ButtonBox["equation", ButtonData:>"Fig:FMC2StageStationarity", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Fig:FMC2StageStationarity"], " is a function only of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], ", whereas the left hand side is a function of both ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], ", where ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " appears only in the ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(2, 1\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"]}], ")"}], TraditionalForm]]], " factor. Because ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " appears only on the left hand side of this equation, its effect must \ somehow vanish." }], "Text"], Cell[TextData[{ "There are two types of solution in which the dependence on ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " vanishes:" }], "Text"], Cell[TextData[{ "1. ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "=", "0"}], TraditionalForm]]], ". The dependence of the left hand side of ", ButtonBox["equation", ButtonData:>"Fig:FMC2StageStationarity", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Fig:FMC2StageStationarity"], " is suppressed by the zero-valued ", Cell[BoxData[ FormBox[ RowBox[{\(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], " factor. Any legal probability ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " is thus permitted for values of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " that have ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "=", "0"}], TraditionalForm]]], ". This solution may be eliminated because for those ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " that have ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "=", "0"}], TraditionalForm]]], " there is no need to define ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " anyway." }], "Text"], Cell[TextData[{ "2. ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_0\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], ">", "0"}], TraditionalForm]]], ". Note that the variance-like factor ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], "0"], RowBox[{\(\(P\&~\)\_\(0, 2\)\), "(", RowBox[{ SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], "|", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "2"]}], ")"}], SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}]}], "-", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"], "\[GreaterEqual]", "0"}], TraditionalForm]]], ", so there are two situations to consider:" }], "Text"], Cell[TextData[{ "a. ", Cell[BoxData[ \(TraditionalForm\`\[LeftDoubleBracketingBar]\[CenterEllipsis]\ \[RightDoubleBracketingBar]\^2 > 0\)]], ". In order to suppress the ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " dependence of the left hand side of ", ButtonBox["equation", ButtonData:>"Fig:FMC2StageStationarity", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Fig:FMC2StageStationarity"], " ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " must be uniquely determined by the value of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], ". So ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "=", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "=", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], "." }], "Text"], Cell[TextData[{ "b. ", Cell[BoxData[ \(TraditionalForm\`\[LeftDoubleBracketingBar]\[CenterEllipsis]\ \[RightDoubleBracketingBar]\^2 = 0\)]], ". Apparently, any legal probability ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], TraditionalForm]]], " is permitted. However, any ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}], "\[NotEqual]", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], " will guarantee that the variance-like factor ", Cell[BoxData[ \(TraditionalForm\`\[LeftDoubleBracketingBar]\[CenterEllipsis]\ \[RightDoubleBracketingBar]\^2 > 0\)]], ", so the only ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " that can possibly survive are ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], "=", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], ". Note that with ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], "=", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], " it is still possible that ", Cell[BoxData[ \(TraditionalForm\`\[LeftDoubleBracketingBar]\[CenterEllipsis]\ \[RightDoubleBracketingBar]\^2 > 0\)]], ", in which case this solution can be eliminated." }], "Text"], Cell[TextData[{ "The only solution that remains is ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], "=", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], ". This result establishes the fact that the replacement of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " by ", Cell[BoxData[ FormBox[ RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}], TraditionalForm]]], " used in ", ButtonBox["equation", ButtonData:>"Eq:WinnerTakeAll", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:WinnerTakeAll"], " (and in ", ButtonBox["equation", ButtonData:>"Eq:WinnerTakeAllSplit", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:WinnerTakeAllSplit"], ", in the case of coupled FMCs) emerges naturally from minimising ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in the function space of probabilities ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], "." }], "Text"], Cell[TextData[{ "It is important to note that the choice of a Euclidean error function is \ sufficient (but not necessary) for ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], "=", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], " to emerge. Also, note that it is not true in general that optimisation of \ ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " leads to ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], "=", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], ". For instance, this is the case when ", Cell[BoxData[ FormBox[ RowBox[{"||", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}], \( || \^2\)}], TraditionalForm]]], " is replaced by either of the following two functional forms" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], "-", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"]}], "\[RightDoubleBracketingBar]"}], "2"], "\[LongRightArrow]", RowBox[{"{", GridBox[{ { RowBox[{ RowBox[{"A", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], "+", RowBox[{"B", "(", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], ")"}]}], " ", \(counterexample\ 1\)}, { RowBox[{ RowBox[{"A", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}], RowBox[{"B", "(", SubscriptBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "\[Prime]"], "0"], ")"}]}], " ", \(counterexample\ 2\)} }]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, CounterIncrements->"NumberedEquation"], Cell["\<\ Although these might not be considered to be sensible error functions, the \ fact that counterexamples exist is in itself important.\ \>", "Text"], Cell["These counterexamples may be described briefly as follows:", "Text", CellTags->"Para:CounterExamples"], Cell[TextData[{ "1. Counterexample 1 leads to a ", Cell[BoxData[ \(TraditionalForm\`D\)]], " which has no dependence on ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], ". Optimisation of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " will allow any legal probability, so ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], "\[NotEqual]", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], " is permitted." }], "Text"], Cell[TextData[{ "2. Counterexample 2 is more complicated to analyse. However, it is \ possible to show that ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], ", when viewed as a matrix, has a block diagonal structure. This type of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], TraditionalForm]]], " does not imply a deterministic relationship between ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], ", so ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_\(1, 0\)\), RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "|", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"]}], ")"}]}], "\[NotEqual]", RowBox[{"\[Delta]", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "-", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "0"], ")"}]}], "]"}]}], TraditionalForm]]], " is permitted." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["Acknowledgements", "Section"], Cell["\<\ The author is indebted to the following people for critically reading this \ paper: Eric Jakeman, David Lowe, Graeme Mitchison.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell["References", "Section"], Cell[TextData[{ "Farvardin N, 1990, A study of vector quantisation for noisy channels, ", StyleBox["IEEE Trans. IT", FontSlant->"Italic"], ", ", StyleBox["36", FontWeight->"Bold"], ", 799-809." }], "Reference", CellTags->"Ref:Farvardin1990"], Cell[TextData[{ "Farvardin N and Vaishampayan V, 1991, On the performance and complexity of \ channel-optimised VQs, ", StyleBox["IEEE Trans. IT", FontSlant->"Italic"], ", ", StyleBox["37", FontWeight->"Bold"], ", 155-160." }], "Reference", CellTags->"Ref:FarvardinVaishampayan1991"], Cell["\<\ Kohonen T, 1984, Self organisation and associative memory, Springer-Verlag.\ \>", "Reference", CellTags->"Ref:Kohonen1984"], Cell[TextData[{ "Kumazawa H, Kasahara M and Namekawa T, 1984, A construction of VQs for \ noisy channels, ", StyleBox["Electronics and Engineering in Japan", FontSlant->"Italic"], ", ", StyleBox["67B", FontWeight->"Bold"], ", 39-47." }], "Reference", CellTags->"Ref:KumazawaKasaharaNamekawa1984"], Cell[TextData[{ "Linde Y, Buzo A and Gray R M, 1980, An algorithm for vector quantiser \ design, ", StyleBox["IEEE Trans. COM", FontSlant->"Italic"], ", ", StyleBox["28", FontWeight->"Bold"], ", 84-95." }], "Reference", CellTags->"Ref:LindeBuzoGray1980"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/ieeenn89/ieeenn89.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell S P, 1989a, Self-organisation: a derivation from first principles \ of a class of learning algorithms, ", StyleBox["Proc. 3rd IEEE Int. Joint Conf. on Neural Networks", FontSlant->"Italic"], ", Washington DC, ", StyleBox["2", FontWeight->"Bold"], ", 495-498." }], "Reference", CellTags->"Ref:Luttrell1989a"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/hiervq/hiervq.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell S P, 1989b, Hierarchical vector quantisation, ", StyleBox["Proc. IEE Part I", FontSlant->"Italic"], ", ", StyleBox["136", FontWeight->"Bold"], ", 405-413." }], "Reference", CellTags->"Ref:Luttrell1989b"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/ieenn89/ieenn89.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell S P, 1989c, Hierarchical self-organising networks, ", StyleBox["Proc. 1st IEE Conf. on Artificial Neural Networks", FontSlant->"Italic"], ", London, 2-6." }], "Reference", CellTags->"Ref:Luttrell1989c"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/compress/compress.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell S P, 1989d, Image compression using a multilayer neural network, \ ", StyleBox["Patt. Recog. Lett.", FontSlant->"Italic"], ", ", StyleBox["10", FontWeight->"Bold"], ", 1-7." }], "Reference", CellTags->"Ref:Luttrell1989d"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/tvq/tvq.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell S P, 1990, Derivation of a class of training algorithms, ", StyleBox["IEEE Trans. NN", FontSlant->"Italic"], ", ", StyleBox["1", FontWeight->"Bold"], ", 229- 232." }], "Reference", CellTags->"Ref:Luttrell1990"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/density/density.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell S P, 1991a, Code vector density in topographic mappings: scalar \ case, ", StyleBox["IEEE Trans. NN", FontSlant->"Italic"], ", ", StyleBox["2", FontWeight->"Bold"], ", 427-436." }], "Reference", CellTags->"Ref:Luttrell1991a"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/ieenn91/ieenn91.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell S P, 1991b, Self-supervised training of hierarchical VQs, ", StyleBox["Proc. 2nd IEE Conf. on Artificial Neural Networks", FontSlant->"Italic"], ", Bournemouth, 5-9." }], "Reference", CellTags->"Ref:Luttrell1991b"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/selfsup/selfsup.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell S P, 1992, Self-supervision in multilayer adaptive networks, ", StyleBox["Proc. IEE Part F", FontSlant->"Italic"], ", ", StyleBox["139", FontWeight->"Bold"], "(6), 371-377." }], "Reference", CellTags->"Ref:Luttrell1992"], Cell[TextData[{ "Ritter H, 1991, Asymptotic level density for a class of vector \ quantisation processes, ", StyleBox["IEEE Trans. NN", FontSlant->"Italic"], ", ", StyleBox["2", FontWeight->"Bold"], ", 173-175." }], "Reference", CellTags->"Ref:Ritter1991"], Cell[TextData[{ "Yair E, Zeger K and Gersho A, 1992, Competitive learning and soft \ competition for vector quantiser design, ", StyleBox["IEEE Trans. SP", FontSlant->"Italic"], ", ", StyleBox["40", FontWeight->"Bold"], ", 294-309." }], "Reference", CellTags->"Ref:YairZegerGersho1992"] }, Closed]] }, Open ]] }, FrontEndVersion->"5.0 for Microsoft Windows", ScreenRectangle->{{0, 1280}, {0, 941}}, WindowSize->{665.375, 641}, WindowMargins->{{307.25, Automatic}, {Automatic, 50}}, Magnification->1, StyleDefinitions -> "Report.nb" ] (******************************************************************* Cached data follows. If you edit this Notebook file directly, not using Mathematica, you must remove the line containing CacheID at the top of the file. The cache data will then be recreated when you save this file from within Mathematica. *******************************************************************) (*CellTagsOutline CellTagsIndex->{ "Sect:Introduction"->{ Cell[24092, 742, 111, 5, 70, "Section 1", CellTags->"Sect:Introduction"]}, "Eq:Objective1"->{ Cell[25388, 782, 1066, 30, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective1"]}, "Eq:Objective2"->{ Cell[32037, 1002, 1129, 32, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective2"]}, "Eq:Objective3"->{ Cell[35774, 1130, 1771, 47, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective3"]}, "Fig:MinimumDistortionEncoding"->{ Cell[46917, 1475, 297, 10, 70, "NumberedFigure", CellTags->"Fig:MinimumDistortionEncoding"]}, "Eq:Objective4"->{ Cell[56373, 1770, 2011, 54, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective4"]}, "Eq:Objective5"->{ Cell[60225, 1889, 2760, 70, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Objective5"]}, "Sect:BasicTheory"->{ Cell[69022, 2144, 108, 5, 70, "Section", CellTags->"Sect:BasicTheory"]}, "Fig:FoldedMarkovChain"->{ Cell[70594, 2188, 289, 10, 70, "NumberedFigure", CellTags->"Fig:FoldedMarkovChain"]}, "Eq:BayesTheorem"->{ Cell[74658, 2316, 1090, 31, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:BayesTheorem"]}, "Eq:ProbabilityFMC"->{ Cell[77601, 2409, 3672, 100, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:ProbabilityFMC"]}, "Eq:Marginal"->{ Cell[81764, 2529, 2324, 65, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:Marginal"]}, "Sect:1and2StageFoldedMarkovChains"->{ Cell[84823, 2621, 148, 5, 70, "Section", CellTags->"Sect:1and2StageFoldedMarkovChains"]}, "Fig:FMC1Stage"->{ Cell[86552, 2678, 281, 10, 70, "NumberedFigure", CellTags->"Fig:FMC1Stage"]}, "Eq:FMC1Stage0"->{ Cell[89919, 2788, 3286, 84, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage0"]}, "Eq:FMC1Stage1"->{ Cell[95212, 2937, 2795, 68, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage1"]}, "Eq:FMC1Stage1a"->{ Cell[99063, 3038, 3170, 69, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage1a"]}, "Eq:FMC1Stage2"->{ Cell[102280, 3111, 3056, 69, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage2"]}, "Eq:FMC1Stage3"->{ Cell[106893, 3235, 3026, 68, 70, "NumberedEquation", CounterIncrements->"NumberedEquation", CellTags->"Eq:FMC1Stage3"]}, "Eq:FMC1Stage4"->{ Cell[116750, 3522