(************** Content-type: application/mathematica ************** CreatedBy='Mathematica 5.0' Mathematica-Compatible Notebook This notebook can be used with any Mathematica-compatible application, such as Mathematica, MathReader or Publicon. The data for the notebook starts with the line containing stars above. To get the notebook into a Mathematica-compatible application, do one of the following: * Save the data starting with the line of stars above into a file with a name ending in .nb, then open the file inside the application; * Copy the data starting with the line of stars above to the clipboard, then use the Paste menu command inside the application. Data for notebooks contains only printable 7-bit ASCII and can be sent directly in email or through ftp in text mode. Newlines can be CR, LF or CRLF (Unix, Macintosh or MS-DOS style). NOTE: If you modify the data for this notebook not in a Mathematica- compatible application, you must delete the line below containing the word CacheID, otherwise Mathematica-compatible applications may try to use invalid cache data. For more information on notebooks and Mathematica-compatible applications, contact Wolfram Research: web: http://www.wolfram.com email: info@wolfram.com phone: +1-217-398-0700 (U.S.) Notebook reader applications are available free of charge from Wolfram Research. *******************************************************************) (*CacheID: 232*) (*NotebookFileLineBreakTest NotebookFileLineBreakTest*) (*NotebookOptionsPosition[ 168277, 4916]*) (*NotebookOutlinePosition[ 175332, 5135]*) (* CellTagsIndexPosition[ 173813, 5082]*) (*WindowFrame->Normal*) Notebook[{ Cell[CellGroupData[{ Cell["Notes", "Section 1"], Cell[CellGroupData[{ Cell["Editorial Changes", "Subsection"], Cell[TextData[{ "Citations \"", Cell[BoxData[ \(TraditionalForm\`\[CenterEllipsis]\^m\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`\[CenterEllipsis]\ [m]\)]], "\"." }], "Text"], Cell["\<\ Equations (26)-(28) erroneously were labelled as equations (27)-(29) in the \ original paper. Also, equation (29) was labelled as equation (26) in the \ original paper. I assume that I had hardwired the equation numbers, and that \ I then created appendix 6.2 by simply moving the equation from the body text \ without remembering to renumber it.\ \>", "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ref:Luttrell1991b", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", StyleBox["SPIE Conf. on Adaptive Signal Processing", FontSlant->"Italic"], "\" changed to \"", StyleBox["Proc. SPIE Conf. on Adaptive Signal Processing", FontSlant->"Italic"], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:19", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:22", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:24", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:29", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "Equation reformatted to make it clearer." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:27", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]]}]], SubscriptBox[ StyleBox[ RowBox[{"\[Bullet]", StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"]}]], RowBox[{ StyleBox["L", FontSlant->"Plain"], "+", "1"}]]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]]}]], ".", SubscriptBox[ StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox["L", FontSlant->"Plain"], "+", "1"}]]}], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:28", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["r", FontSlant->"Plain"]]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["r", FontSlant->"Plain"]]}]], SubscriptBox[ StyleBox[ RowBox[{"\[Bullet]", StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"]}]], RowBox[{ StyleBox["r", FontSlant->"Plain"], "+", "1"}]]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["r", FontSlant->"Plain"]]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["r", FontSlant->"Plain"]]}]], ".", SubscriptBox[ StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox["r", FontSlant->"Plain"], "+", "1"}]]}], TraditionalForm]]], "\", and \"", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[\(N + 1\), FontSlant->"Plain"]], "=", "\[AlignmentMarker]", "0"}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[\(N + 1\), FontSlant->"Plain"]], "=", "\[AlignmentMarker]", StyleBox["0", FontWeight->"Bold"]}], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["PROBLEM", ButtonData:>"Ed:Change1", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "Column alignments are not the same as in the original paper." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["Adaptive Bayesian Networks", "Title"], Cell["\<\ S P Luttrell Defence Research Agency St Andrews Rd, Malvern, WORCS, WR14 3PS Janet: luttrell@uk.mod.hermes Internet: luttrell%hermes.mod.uk@relay.mod.uk\ \>", "Author"], Cell["\<\ This appeared in Proceedings of the SPIE Conference on Adaptive and Learning \ Systems, Orlando, 140-151, 1992.\ \>", "Text"], Cell["\<\ ABSTRACT The theory of adaptive Bayesian networks is summarised. A detailed discussion \ of the Adaptive Cluster Expansion (ACE) network is presented. ACE is a \ scalable Bayesian network designed specifically for high-dimensional \ applications, such as image processing.\ \>", "Abstract"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". INTRODUCTION" }], "Section 1"], Cell[TextData[{ "In the first half of this paper the theory of adaptive Bayesian networks \ is presented. This type of network performs Bayesian inference [", ButtonBox["1", ButtonData:>"Ref:Cox1946", ButtonStyle->"Hyperlink"], ", ", ButtonBox["2", ButtonData:>"Ref:Jeffreys1939", ButtonStyle->"Hyperlink"], "] using a probability density function (PDF) model that it learns during a \ training programme. Because a large number of degrees of freedom (or hidden \ variables) need to be integrated out, Bayesian networks can be \ computationally expensive to simulate [", ButtonBox["3", ButtonData:>"Ref:AckleyHintonSejnowski1985", ButtonStyle->"Hyperlink"], "]. In the second half of this paper the Adaptive Cluster Expansion (ACE) \ is offered as a computationally cheaper alternative." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". GENERAL THEORETICAL FRAMEWORK" }], "Section"], Cell[TextData[{ "In this section the principles of adaptive Bayesian networks are \ discussed. The emphasis is on clarifying the underlying principles and \ exposing hidden approximations. The notation used in this paper can be found \ in the appendix. It is important to realise that there are two different \ types of PDF used: Bayesian (denoted as ", Cell[BoxData[ FormBox[ StyleBox["Q", FontSlant->"Plain"], TraditionalForm]]], "), and frequentist (denoted as ", Cell[BoxData[ FormBox[ StyleBox["P", FontSlant->"Plain"], TraditionalForm]]], "). ", Cell[BoxData[ FormBox[ StyleBox["Q", FontSlant->"Plain"], TraditionalForm]]], " is used to denote a Bayesian model PDF, whereas ", Cell[BoxData[ FormBox[ StyleBox["P", FontSlant->"Plain"], TraditionalForm]]], " is used to denote a frequency derived from a training set (used only \ where a full Bayesian analysis is impractical)." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Making predictions from a model" }], "Subsection"], Cell["\<\ The fundamental problem is this: given a training set, and a model PDF, use \ Bayesian methods to make predictions about test data. Bayesian inference \ leaves no freedom of action once the problem is thus specified, and it yields \ the predictive model\ \>", "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}], "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}]}]}]}], " ", "exact"}, { RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}], "\[TildeTilde]", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"], ")"}]}], ")"}]}], " ", "approximate"} }], TraditionalForm]], "NumberedEquation", TextAlignment->Left, GridBoxOptions->{ColumnAlignments->{Left}}, CellTags->"Eq:1"], Cell["\<\ The exact result is an application of a hidden variables model, in which the \ model parameters are integrated out thus\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], ",", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}], "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], ",", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:2"], Cell["\<\ The quality of the exact result is limited only by the validity of the model. \ If there are several contending models, perhaps none of which is actually \ correct, then a single winner could be selected by, for instance, choosing \ the model that yielded the greatest probability of generating the data.\ \>", "Text"], Cell["\<\ On the other hand, the approximate result avoids the computationally \ expensive integral over parameters. There is a variety of such methods, \ including\ \>", "Text"], Cell[BoxData[ FormBox[GridBox[{ {\(1. \ maximum\ likelihood\), " ", RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"], ")"}], "=", RowBox[{GridBox[{ {\(arg\ max\)}, { StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]} }, ColumnAlignments->{Center}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}]}, {\(2. \ maximum\ posterior\ probability\), " ", RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"], ")"}], "=", RowBox[{GridBox[{ {\(arg\ max\)}, { StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]} }, ColumnAlignments->{Center}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}]}]}]}, {\(3. \ maximum\ discriminatory\ likelihood\), " ", RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"], ")"}], "=", RowBox[{GridBox[{ {\(arg\ max\)}, { StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]} }, ColumnAlignments->{Center}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "out", "-"], "\[VerticalSeparator]", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "in", "-"]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}]} }], TraditionalForm]], "NumberedEquation", TextAlignment->Left, GridBoxOptions->{ColumnAlignments->{Left}}, CellTags->"Eq:3"], Cell[TextData[{ "The choice of approximation scheme can be used to influence the \ capabilities of the predictive model. If the posterior probability over \ parameters is highly localised, then ", ButtonBox["schemes 1 and 2", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], " are good approximations. On the other hand, the data might be partitioned \ into separate input and output spaces, and the model might need to be \ optimised for computing the conditional probability of the output given the \ input (i.e. to discriminate between alternative outputs). This would be a job \ for ", ButtonBox["scheme 3", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], "." }], "Text"], Cell[TextData[{ "In all schemes, whether exact or approximate, there is the problem of \ missing data. The exact method is unaffected by this, because missing data \ can be treated as hidden variables, and eliminated by integration. Whether or \ not an approximate method survives the problem of missing data depends on its \ structure. For instance, if the output data were omitted, then ", ButtonBox["scheme 3", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], " would be meaningless. There are many ways in which an approximate method \ could fail due to violations of its underlying assumptions. The moral is to \ construct the approximate method to suit the data, rather than the other way \ around." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Visible and hidden variables" }], "Subsection"], Cell["\<\ The data space may be augmented by appending an additional unobserved vector \ to the original data vector. In this case, the data vector comprises \ \"visible variables\", whereas the unobserved vector comprises \"hidden \ variables\". The model PDF then becomes\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "=", RowBox[{ RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", RowBox[{ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]}], ")"}]}]}], "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:4"], Cell["\<\ Hidden variables can always be appended to a model, at the cost of additional \ theoretical and computational effort.\ \>", "Text"], Cell["The prediction equation then becomes", "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}], "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], "+"], SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], "-"], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], ",", RowBox[{ SuperscriptBox[ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", RowBox[{ SuperscriptBox[ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "-"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}]}], ")"}]}]}]}], " ", "exact"}, { RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}], "\[TildeEqual]", RowBox[{"\[Integral]", RowBox[{ SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], "+"], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], ",", RowBox[{ SuperscriptBox[ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"], ")"}]}]}], ")"}]}]}]}], " ", "approximate"} }], TraditionalForm]], "NumberedEquation", TextAlignment->Left, GridBoxOptions->{ColumnAlignments->{Left}}, CellTags->"Eq:5"], Cell["where the notation used is self-explanatory.", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Logarithmic likelihood and relative entropy" }], "Subsection"], Cell[TextData[{ "If ", Cell[BoxData[ FormBox[ StyleBox["N", FontSlant->"Plain"], TraditionalForm]]], " data samples are drawn independently from the training set, then the \ logarithmic likelihood may be written as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["L", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[Congruent]", RowBox[{"log", "(", RowBox[{ UnderoverscriptBox["\[Product]", RowBox[{ StyleBox["k", FontSlant->"Plain"], "=", "1"}], StyleBox["N", FontSlant->"Plain"]], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["k", FontSlant->"Plain"]], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}], ")"}], "\[TildeTilde]", RowBox[{ StyleBox["N", FontSlant->"Plain"], RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{"log", "(", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], ")"}]}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:6"], Cell[TextData[{ "The integration measure ", Cell[BoxData[ FormBox[ StyleBox["P", FontSlant->"Plain"], TraditionalForm]]], " is a frequentist (i.e. non-Bayesian) PDF; it is used as a convenient \ shorthand notation, and will not enter into any Bayesian manipulations." }], "Text"], Cell[TextData[{ "On the other hand, the relative entropy (between ", Cell[BoxData[ FormBox[ StyleBox["P", FontSlant->"Plain"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ StyleBox["Q", FontSlant->"Plain"], TraditionalForm]]], ") is defined as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["G", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[Congruent]", RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{"log", "(", FractionBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]], ")"}]}]}]}], "\[LessEqual]", "0"}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:7"], Cell[TextData[{ "Relative entropy is zero if, and only if, ", Cell[BoxData[ FormBox[ StyleBox["P", FontSlant->"Plain"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ StyleBox["Q", FontSlant->"Plain"], TraditionalForm]]], " are identical, so it may be used as a fitness criterion for comparing ", Cell[BoxData[ FormBox[ StyleBox["P", FontSlant->"Plain"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ StyleBox["Q", FontSlant->"Plain"], TraditionalForm]]], ". Relative entropy is the logarithm of the probability (per sample) that \ samples taken from ", Cell[BoxData[ FormBox[ StyleBox["Q", FontSlant->"Plain"], TraditionalForm]]], " have the frequency of occurrence specified by ", Cell[BoxData[ FormBox[ StyleBox["P", FontSlant->"Plain"], TraditionalForm]]], "." }], "Text"], Cell["These results can be combined to yield", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["L", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[TildeTilde]", RowBox[{ RowBox[{ StyleBox["N", FontSlant->"Plain"], StyleBox[" ", FontSlant->"Plain"], RowBox[{ StyleBox["G", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "+", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{"log", "(", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], ")"}]}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:8"], Cell[TextData[{ "The second term on the right hand side does not depend on the model \ parameters, so relative entropy is (up to an additive constant) approximately \ equal to logarithmic likelihood. In the limit of a large number of training \ samples, relative entropy maximisation is equivalent to maximum likelihood \ maximisation (i.e. it corresponds to approximate ", ButtonBox["scheme 1", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], ")." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". SOME ADAPTIVE NETWORKS REVISITED" }], "Section"], Cell["\<\ In this section a brief review of some familiar adaptive Bayesian networks is \ presented.\ \>", "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Mixture distribution" }], "Subsection"], Cell["\<\ A mixture distribution is a hidden variables model with a PDF of the form\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "=", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["c", FontSlant->"Plain"], "=", "1"}], StyleBox["M", FontSlant->"Plain"]], RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["c", FontSlant->"Plain"]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["c", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:9"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ StyleBox["c", FontSlant->"Plain"], TraditionalForm]], FontSlant->"Italic"], " is a \"class label\" which is a discrete-valued hidden variable. If the \ dependence on parameters is modified as follows" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["c", FontSlant->"Plain"]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["c", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "\[LongRightArrow]", "\[AlignmentMarker]", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["c", FontSlant->"Plain"]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", StyleBox["c", FontSlant->"Plain"], ")"}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "\[LongRightArrow]", "\[AlignmentMarker]", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ",", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", StyleBox["c", FontSlant->"Plain"], ")"}]}], ")"}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:10"], Cell[TextData[{ "where the prior probability factor itself comprises ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["M", FontSlant->"Plain"], "-", "1"}], TraditionalForm]]], " of the parameters (i.e. it is non-parametric), then the iterative \ re-estimation prescription [", ButtonBox["4", ButtonData:>"Ref:Baum1972", ButtonStyle->"Hyperlink"], ", ", ButtonBox["5", ButtonData:>"Ref:DempsterLairdRubin1977", ButtonStyle->"Hyperlink"], "] for maximising relative entropy becomes" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ OverscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "^"], "=", "\[AlignmentMarker]", RowBox[{GridBox[{ {\(arg\ max\)}, { StyleBox[ SuperscriptBox["s", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]} }], RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["c", FontSlant->"Plain"], "=", "1"}], StyleBox["M", FontSlant->"Plain"]], RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["c", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{"log", "(", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["c", FontSlant->"Plain"]}], ",", StyleBox[ SuperscriptBox["s", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], ")"}]}]}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ OverscriptBox[ StyleBox["Q", FontSlant->"Plain"], "^"], "(", StyleBox["c", FontSlant->"Plain"], ")"}], "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["c", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:11"], Cell[TextData[{ "The structure of a mixture distribution network is shown in ", ButtonBox["Figure", ButtonData:>"Fig:1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:1"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/spie92/fig1.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:1"], Cell["\<\ Diagram showing a network suitable for computing a mixture distribution. The \ input layer receives the data vector, and the output layer produces the \ corresponding class conditional probabilities. The network parameters are \ located as indicated.\ \>", "Caption"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Hidden Markov model" }], "Subsection"], Cell["\<\ A hidden Markov model is a mixture distribution whose hidden variables have a \ memory of their previous state in time. Thus the mixture distribution PDF is \ modified to become\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ",", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], RowBox[{ StyleBox["t", FontSlant->"Plain"], "-", "1"}]]}], ")"}]}], ")"}], "=", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]], "=", "1"}], StyleBox["M", FontSlant->"Plain"]], RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], RowBox[{ StyleBox["t", FontSlant->"Plain"], "-", "1"}]]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:12"], Cell[TextData[{ "where a discrete time index ", Cell[BoxData[ FormBox[ StyleBox["t", FontSlant->"Plain"], TraditionalForm]]], " now appears, and the prior probability is replaced by a transition \ matrix. The full joint PDF for a time series of data is then given by" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ",", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], "\[Tau]"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], RowBox[{ StyleBox["\[Tau]", FontSlant->"Plain"], "-", "1"}]]}], ")"}]}], ")"}], "=", RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], "0"], ")"}], RowBox[{ UnderoverscriptBox["\[Product]", RowBox[{ StyleBox["t", FontSlant->"Plain"], "=", "1"}], StyleBox["T", FontSlant->"Plain"]], RowBox[{"(", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]], "=", "1"}], StyleBox["M", FontSlant->"Plain"]], RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], StyleBox["t", FontSlant->"Plain"]], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["c", FontSlant->"Plain"], RowBox[{ StyleBox["t", FontSlant->"Plain"], "-", "1"}]]}], ")"}]}]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:13"], Cell[TextData[{ "where a prior probability which specifies the initial distribution of the \ hidden variable ", Cell[BoxData[ FormBox[ StyleBox["c", FontSlant->"Plain"], TraditionalForm]]], " is included. The re-estimation prescription may be used to optimise the \ model parameters." }], "Text"], Cell[TextData[{ "The structure of a hidden Markov model network is shown in ", ButtonBox["Figure", ButtonData:>"Fig:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:2"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/spie92/fig2.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:2"], Cell["\<\ Diagram showing a network suitable for computing a hidden Markov model. As in \ the mixture distribution network, the input layer receives the data vector, \ and the output layer produces the corresponding class conditional \ probabilities. The network parameters are located as indicated. The output \ layer is influenced by its own previous output values, which endows the \ network with discrete time dynamics.\ \>", "Caption"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Boltzmann machine" }], "Subsection"], Cell[TextData[{ "Gibbs distributions [", ButtonBox["6", ButtonData:>"Ref:KindermannSnell1980", ButtonStyle->"Hyperlink"], "] (or, equivalently, Markov random fields) are a maximum entropy family of \ model PDF's with the form" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "=", "\[AlignmentMarker]", RowBox[{ FractionBox["1", RowBox[{"Z", "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], RowBox[{"exp", "(", RowBox[{"-", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ".", RowBox[{ StyleBox["U", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}], ")"}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{"where", " ", RowBox[{"Z", "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{"exp", "(", RowBox[{"-", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ".", RowBox[{ StyleBox["U", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}], ")"}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:14"], Cell["\<\ which depend exponentially on a sum of \"potentials\". Hidden variables may \ readily be introduced into such models.\ \>", "Text"], Cell[TextData[{ "Because Gibbs distributions do not generally have the simple structure of \ mixture distributions or hidden Markov models, it is not usually possible to \ use the re-estimation prescription to optimise them. A relative entropy \ gradient ascent algorithm could be used instead, based on the result [", ButtonBox["7", ButtonData:>"Ref:Luttrell1989a", ButtonStyle->"Hyperlink"], "]" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", RowBox[{"G", "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], RowBox[{"\[PartialD]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], "=", RowBox[{ RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", RowBox[{ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]}], ")"}], RowBox[{ StyleBox["U", FontWeight->"Bold", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}], "-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["P", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["U", FontWeight->"Bold", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:15"], Cell[TextData[{ "This leads to a generalised form of the Boltzmann machine training \ algorithm [", ButtonBox["3", ButtonData:>"Ref:AckleyHintonSejnowski1985", ButtonStyle->"Hyperlink"], "], which optimises the model in a way that depends on the difference \ between the \"free\" average and the \"clamped\" average of the potentials." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". ADAPTIVE CLUSTER EXPANSION (ACE)" }], "Section"], Cell[TextData[{ "In this section an up-to-date discussion of the Adaptive Cluster Expansion \ (ACE) method [", ButtonBox["7", ButtonData:>"Ref:Luttrell1989a", ButtonStyle->"Hyperlink"], "] is presented, followed by a detailed theoretical analysis of a simple \ ACE network based on mixture distributions. A good example of the application \ of this approach is the anomaly detector [", ButtonBox["8", ButtonData:>"Ref:Luttrell1990", ButtonStyle->"Hyperlink"], ", ", ButtonBox["9", ButtonData:>"Ref:Luttrell1991a", ButtonStyle->"Hyperlink"], "]. Other uses (i.e. not PDF models) of the ACE approach have also been \ published [", ButtonBox["10", ButtonData:>"Ref:Luttrell1989b", ButtonStyle->"Hyperlink"], ", ", ButtonBox["11", ButtonData:>"Ref:Luttrell1989c", ButtonStyle->"Hyperlink"], "]." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ACE philosophy" }], "Subsection"], Cell["\<\ One of the major problems with Bayesian networks with hidden variables is \ their tendency to consume large amounts of computing resources when \ evaluating the integration over hidden variables (including model parameters) \ in the exact prediction equation\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}], "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], "+"], SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], "-"], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], ",", RowBox[{ SuperscriptBox[ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", RowBox[{ SuperscriptBox[ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "-"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:16"], Cell["\<\ These numerical problems can be especially acute if Monte Carlo simulations \ are used.\ \>", "Text"], Cell["\<\ The ACE approach factorises this equation into the following form\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}], "=", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["+", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "2"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["+", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["J", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["+", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ")"}], ",", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "2"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["+", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[",", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:17"], Cell[TextData[{ "where the first two factors model the PDF in two independent subspaces (or \ clusters), and the third factor models the joint PDF of transformed versions \ of the clusters. In the ", Cell[BoxData[ FormBox[ StyleBox["J", FontSlant->"Plain"], TraditionalForm]]], "-factor, the choice of cluster transformations is arbitrary, though the \ choice affects the performance of the prediction equation. To ensure that the \ overall normalisation is correct, it is necessary to write the ", Cell[BoxData[ FormBox[ StyleBox["J", FontSlant->"Plain"], TraditionalForm]]], "-factor in the form" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["J", FontSlant->"Plain"], "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "2"], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[",", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "=", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "12"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["y", StyleBox["1", FontWeight->"Plain"], "+"], FontWeight->"Bold", FontSlant->"Plain"], StyleBox[",", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox[ SubsuperscriptBox["y", StyleBox["2", FontWeight->"Plain"], StyleBox["+", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["y", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[",", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[ SubsuperscriptBox["y", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["y", StyleBox["1", FontWeight->"Plain"], "+"], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "2"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["y", StyleBox["2", FontWeight->"Plain"], "+"], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:18"], Cell["\<\ where the numerator models the PDF of the transformed clusters, and the \ denominator is the product of the transformed PDF's of the original clusters, \ defined by\ \>", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["y", StyleBox["1", FontWeight->"Plain"], "+"], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontWeight->"Plain"], "x"}]], StyleBox["1", FontWeight->"Plain"], StyleBox["+", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], "+"], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ StyleBox[ SubsuperscriptBox["y", StyleBox["1", FontWeight->"Plain"], "+"], FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], "(", StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], "+"], FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}], "a"}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"and", " ", "similarly", " ", RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "2"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["y", StyleBox["2", FontWeight->"Plain"], "+"], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:19"], Cell["\<\ The above factorisation cannot be derived from the original prediction \ equation; it has to be assumed, and justified as follows. Suppose there are \ four Bayesians, each of whom has partial access to the training data. \ Bayesian 1 has access to cluster 1, Bayesian 2 has access to cluster 2, and \ Bayesian 3 has access to the transformed cluster pairs. Bayesians 1-3 can \ each independently form an exact prediction equation for his own (sub)space. \ Bayesian 4 can then use these three results (and no further information) to \ form a uniquely consistent combination, which is the factorised prediction \ equation stated earlier.\ \>", "Text"], Cell["\<\ Finally, each of the factors can be modelled by any of the methods discussed \ thus far: exact or approximate, and with or without additional hidden \ variables.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Approximate ACE: remove the integrals over parameters" }], "Subsection"], Cell["\<\ The above ACE expression for two cluster may be approximated as\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], ")"}], "\[TildeTilde]", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["+", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Bold", FontSlant->"Plain"]}]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "2"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["+", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], FractionBox[ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "12"], "(", RowBox[{ StyleBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"], "+"], ")"}], FontWeight->"Bold", FontSlant->"Plain"], StyleBox[",", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"], "+"], ")"}], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Bold", FontSlant->"Plain"]}]}]}], ")"}], RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"], "+"], ")"}], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "2"], "(", RowBox[{ StyleBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"], "+"], ")"}], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:20"], Cell[TextData[{ "where the parameter values have been set to their maximum likelihood \ values (i.e. approximate ", ButtonBox["scheme 1", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], ")" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{" ", RowBox[{ RowBox[{"(", "\[NegativeThinSpace]", GridBox[{ { RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Bold", FontSlant->"Plain"]}]}, { StyleBox[ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], FontWeight->"Bold", FontSlant->"Plain"]}, { RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Bold", FontSlant->"Plain"]}]} }], "\[NegativeThinSpace]", ")"}], "=", RowBox[{GridBox[{ {\(arg\ max\)}, { RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}]} }], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["1", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "2"], "(", RowBox[{ StyleBox[ SubsuperscriptBox["x", StyleBox["2", FontWeight->"Plain"], StyleBox["-", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], FractionBox[ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "12"], "(", RowBox[{ StyleBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"], "-"], ")"}], FontWeight->"Bold", FontSlant->"Plain"], StyleBox[",", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"], "-"], ")"}], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}]}], ")"}], RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"], "-"], ")"}], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "2"], "(", RowBox[{ StyleBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"], "-"], ")"}], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:21"], Cell[TextData[{ "Note that the parameters need to be jointly optimised because the term \ arising from the ", Cell[BoxData[ FormBox[ StyleBox["J", FontSlant->"Plain"], TraditionalForm]]], "-factor is not independent of the first two factors." }], "Text"], Cell["\<\ When the amount of training data is large, this result is equivalent to \ relative entropy maximisation. Thus define\ \>", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["G", FontSlant->"Plain"], "1"], "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], "=", "\[AlignmentMarker]", RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], "1"], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], RowBox[{"log", "(", FractionBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubscriptBox["x", StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ")"}]], ")"}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"and", " ", "similarly", " ", RowBox[{ SubscriptBox[ StyleBox["G", FontSlant->"Plain"], "2"], "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["I", FontSlant->"Plain"], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}], ")"}], "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"]}]], "1"], " ", SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"]}]], "2"], " ", RowBox[{"P", "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "2"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}], RowBox[{"log", "(", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "12"], "(", RowBox[{ StyleBox[ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], StyleBox[",", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox[ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}]}], ")"}], RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubscriptBox["s", StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "2"], "(", RowBox[{ StyleBox[ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubscriptBox["s", StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]], ")"}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["I", FontSlant->"Plain"], "0"], "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], "1"], " ", SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], "2"], " ", RowBox[{"P", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}], RowBox[{"log", "(", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}], RowBox[{ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "2"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}]}]], ")"}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:22"], Cell[TextData[{ "to obtain [", ButtonBox["12", ButtonData:>"Ref:Luttrell1991b", ButtonStyle->"Hyperlink"], "]." }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["G", FontSlant->"Plain"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}], ")"}], "=", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["G", FontSlant->"Plain"], "1"], "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], "+", RowBox[{ SubscriptBox[ StyleBox["G", FontSlant->"Plain"], "2"], "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}], "+", RowBox[{ SubscriptBox[ StyleBox["I", FontSlant->"Plain"], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}], ")"}], "-", SubscriptBox[ StyleBox["I", FontSlant->"Plain"], "0"]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:23"], Cell["\<\ The first two terms are the relative entropies evaluated independently for \ each cluster, the third term is (a model of) the mutual information between \ the transformed clusters, and the fourth term is the (constant) mutual \ information between the clusters themselves.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Mixture distribution ACE" }], "Subsection"], Cell["\<\ Although the integrals over parameters were removed above, it is still \ possible to introduce additional hidden variables. Thus model each PDF in the \ ACE expression as a mixture distribution\ \>", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ")"}], "=", "\[AlignmentMarker]", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["c", FontSlant->"Plain"], "=", "1"}], StyleBox["M", FontSlant->"Plain"]], RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], "\[VerticalSeparator]", StyleBox["c", FontSlant->"Plain"]}], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox["c", FontSlant->"Plain"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ")"}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"and", " ", "similarly", " ", RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "2"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "2"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}]}], ")"}], "=", "\[AlignmentMarker]\[AlignmentMarker]", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["c", FontSlant->"Plain"], "=", "1"}], StyleBox["M", FontSlant->"Plain"]], RowBox[{ RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "2"], "\[VerticalSeparator]", StyleBox["c", FontSlant->"Plain"]}], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "12"], "(", RowBox[{ StyleBox["c", FontSlant->"Plain"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}], ")"}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:24"], Cell["\<\ Furthermore, it is possible to define the cluster transformations using \ quantities that are computed by the mixture distribution model. For instance, \ one possible choice is the vector of posterior class probabilities. Omitting \ the cluster suffices, this may be written as\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "=", FractionBox[ RowBox[{"(", RowBox[{ RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["c", FontSlant->"Plain"]}], "=", "1"}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["c", FontSlant->"Plain"], "=", RowBox[{"1", "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]}], ")"}]}], ",", RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["c", FontSlant->"Plain"]}], "=", "2"}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["c", FontSlant->"Plain"], "=", RowBox[{"2", "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]}], ")"}]}], ",", "\[CenterEllipsis]"}], ")"}], RowBox[{"\[AlignmentMarker]", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["c", FontSlant->"Plain"], "=", "1"}], StyleBox["M", FontSlant->"Plain"]], RowBox[{ RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["c", FontSlant->"Plain"]}], ",", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["Q", FontSlant->"Plain"], "(", RowBox[{ StyleBox["c", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}]}]]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:25"], Cell["\<\ This choice is non-Bayesian, because a probability is being used as if it \ were a state vector, which is not permitted for Bayesian probabilities. \ However, because it is being used only as a cluster transformation, and not \ in any further Bayesian manipulations, this procedure is allowed.\ \>", "Text"], Cell[TextData[{ "The structure of an ACE network suitable for computing the above results \ is shown in ", ButtonBox["figure", ButtonData:>"Fig:3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:3"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/spie92/fig3.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:3"], Cell["\<\ Diagram showing an ACE network. The second stage receives as input two \ vectors of posterior class probabilities computed in the first stage.\ \>", "Caption"], Cell["\<\ In the appendix, expressions for the derivatives of the relative entropy are \ presented, together with a brief discussion on self-supervision.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " General ACE model" }], "Subsection"], Cell[TextData[{ "By including a ", Cell[BoxData[ FormBox[ StyleBox["J", FontSlant->"Plain"], TraditionalForm]]], "-factor for each cluster transformation, the above ACE network can readily \ be generalised to a tree-structured network with any number of layers. The \ expression for the relative entropy is similarly modified to incorporate the \ additional ", Cell[BoxData[ FormBox[ StyleBox["J", FontSlant->"Plain"], TraditionalForm]]], "-factors." }], "Text"], Cell["\<\ The gradients of the relative entropy may be derived as follows. Introduce \ some simplified notation\ \>", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]], "=", "\[AlignmentMarker]", RowBox[{"input", " ", "to", " ", "layer", " ", StyleBox["L", FontSlant->"Plain"]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]], "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"output", " ", "from", " ", "layer", " ", StyleBox["L", FontSlant->"Plain"]}], "=", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox["L", FontSlant->"Plain"], "+", "1"}]], "=", RowBox[{ RowBox[{"input", " ", "to", " ", "layer", " ", StyleBox["L", FontSlant->"Plain"]}], "+", "1"}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]], "=", "\[AlignmentMarker]", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]], ")"}], "=", RowBox[{ "cluster", " ", "transformations", " ", "in", " ", "layer", " ", StyleBox["L", FontSlant->"Plain"]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ StyleBox["G", FontSlant->"Plain"], "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox["G", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}], "=", \(decomposition\ by\ input\ sample\)}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ StyleBox["G", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "=", "\[AlignmentMarker]\[AlignmentMarker]\[AlignmentMarker]", RowBox[{ RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["L", FontSlant->"Plain"], "=", "1"}], StyleBox["N", FontSlant->"Plain"]], RowBox[{ SubscriptBox[ StyleBox["J", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]], ")"}]}], "=", \(decomposition\ by\ network\ layer\)}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:26"], Cell[TextData[{ "The relative entropy may be differentiated with respect to the parameters \ in network layer ", Cell[BoxData[ FormBox[ StyleBox["L", FontSlant->"Plain"], TraditionalForm]]], " to yield" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", RowBox[{ StyleBox["G", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]]}]], "=", RowBox[{ FractionBox[ RowBox[{"\[PartialD]", RowBox[{ StyleBox[ SubscriptBox["J", StyleBox["L", FontSlant->"Plain"]], FontSlant->"Plain"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]]}]], "+", RowBox[{ FractionBox[ RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["L", FontSlant->"Plain"]]}]], ".", SubscriptBox[ StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox["L", FontSlant->"Plain"], "+", "1"}]]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:27"], Cell[TextData[{ "where the back-propagated signal from layer ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["L", FontSlant->"Plain"], "+", "1"}], TraditionalForm]]], " is obtained by iterating the back-propagation equations" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["r", FontSlant->"Plain"]], "=", "\[AlignmentMarker]", RowBox[{ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", RowBox[{ StyleBox[ SubscriptBox["J", StyleBox["r", FontSlant->"Plain"]], FontSlant->"Plain"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["r", FontSlant->"Plain"]], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["r", FontSlant->"Plain"]]}]], "+", RowBox[{ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["r", FontSlant->"Plain"]]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["r", FontSlant->"Plain"]]}]], ".", SubscriptBox[ StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[\(r + 1\), FontSlant->"Plain"]]}], " ", StyleBox["r", FontSlant->"Plain"]}]}], "=", StyleBox["N", FontSlant->"Plain"]}]}], ",", RowBox[{ StyleBox["N", FontSlant->"Plain"], "-", "1"}], ",", "\[CenterEllipsis]", ",", "3", ",", "2", ",", "1"}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["b", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[\(N + 1\), FontSlant->"Plain"]], "=", "\[AlignmentMarker]", StyleBox["0", FontWeight->"Bold"]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:28"], Cell[TextData[{ "The flow of information needed to compute the above results is shown in ", ButtonBox["figure", ButtonData:>"Fig:4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:4"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/spie92/fig4.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:4"], Cell["\<\ Diagram showing a schematic multilayer ACE network. The flow from \ left-to-right is the forward-propagating data, and the flow from right-to- \ left is the backward-propagating derivatives generated within the network. \ The contributions to the relative entropy and its derivatives are output from \ each network layer, as shown.\ \>", "Caption"], Cell["\<\ It is also possible to replace the mixture distributions used above, by \ hidden Markov models. This is a natural way of introducing temporal behaviour \ into ACE.\ \>", "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". CONCLUSIONS" }], "Section"], Cell["\<\ The theory of adaptive Bayesian networks is very powerful, and contains many \ familiar methods as special cases. The main problem with the Bayesian method, \ when applied rigorously, is the need to perform integrations over hidden \ variables, which can involve lengthy Monte Carlo simulations in some cases. \ The Adaptive Cluster Expansion (ACE) method reviewed in this paper provides \ one possible escape route from the need for such integrations. ACE provides a \ scalable network architecture that can readily be applied to large datasets, \ such as images.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". APPENDIX" }], "Section"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Basic notation" }], "Subsection"], Cell[BoxData[ FormBox[GridBox[{ { StyleBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["k", FontSlant->"Plain"]], GridBoxOptions->{ColumnAlignments->{Right}}], "=", RowBox[{ StyleBox["k", FontSlant->"Plain"], "-", \(th\ vector\ in\ data\ space\)}]}, { StyleBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"], GridBoxOptions->{ColumnAlignments->{Right}}], "=", RowBox[{ RowBox[{"(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ",", "\[CenterEllipsis]", ",", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["N", FontSlant->"Plain"]]}], ")"}], "=", \(past\ data\ \((training\ data)\)\)}]}, { StyleBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "+"], GridBoxOptions->{ColumnAlignments->{Right}}], "=", RowBox[{ RowBox[{"(", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox["N", FontSlant->"Plain"], "+", "1"}]], ",", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox["N", FontSlant->"Plain"], "+", "2"}]], ",", "\[CenterEllipsis]"}], ")"}], "=", \(future\ data\ \((test\ data)\)\)}]}, { StyleBox[ RowBox[{ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], GridBoxOptions->{ColumnAlignments->{Right}}], "=", \(transformation\ of\ data\ vector\)}, { StyleBox[ RowBox[{ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "(", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[PlusMinus]"], ")"}], GridBoxOptions->{ColumnAlignments->{Right}}], "=", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "\[PlusMinus]"], "=", \(transformation\ of\ all\ future/past\ data\)}]}, { StyleBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], GridBoxOptions->{ColumnAlignments->{Right}}], "=", \(hidden\ variables\ vectors\)}, { StyleBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], GridBoxOptions->{ColumnAlignments->{Right}}], "=", \(\(PDF\ of\ data\ \((frequentist\ \*"\"\<\>" real\ world\) \*"\"\<\>"\)\()\)\)}, { RowBox[{ StyleBox["Q", FontSlant->"Plain", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox["(", GridBoxOptions->{ColumnAlignments->{Right}}], RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox["\[VerticalSeparator]", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain", GridBoxOptions->{ColumnAlignments->{Right}}]}], StyleBox[")", GridBoxOptions->{ColumnAlignments->{Right}}]}], "=", \(conditional\ PDF\ of\ data\ given\ a\ parameter\ vector\ \ \((Bayesian\ model)\)\)}, { RowBox[{ StyleBox["Q", FontSlant->"Plain", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox["(", GridBoxOptions->{ColumnAlignments->{Right}}], RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain", GridBoxOptions->{ColumnAlignments->{Right}}], "+"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-"]}], StyleBox[")", GridBoxOptions->{ColumnAlignments->{Right}}]}], "=", \(conditional\ PDF\ of\ future\ data\ given\ past\ data\ \ \((Bayesian\ model)\)\)}, { RowBox[{ StyleBox["L", FontSlant->"Plain", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox["(", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox[")", GridBoxOptions->{ColumnAlignments->{Right}}]}], "=", \(logarithmic\ likelihood\)}, { RowBox[{ StyleBox["G", FontSlant->"Plain", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox["(", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain", GridBoxOptions->{ColumnAlignments->{Right}}], StyleBox[")", GridBoxOptions->{ColumnAlignments->{Right}}]}], "=", \(relative\ entropy\)} }], TraditionalForm]], "Text", GridBoxOptions->{ColumnAlignments->{Left}}, CellTags->"Ed:Change1"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Gradient of relative entropy in mixture distribution ACE" }], "Subsection"], Cell["The gradients of the relative entropy are", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", RowBox[{ SubscriptBox[ StyleBox["G", FontSlant->"Plain"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontSlant->"Plain"]]}]], "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], "1"], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], FractionBox[ RowBox[{\(\[PartialD]log\), " ", RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "1"], "(", RowBox[{ StyleBox[ SubscriptBox["x", StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontSlant->"Plain"]]}]]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"and", " ", "similarly", " ", FractionBox[ RowBox[{"\[PartialD]", RowBox[{ SubscriptBox[ StyleBox["G", FontSlant->"Plain"], "2"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontSlant->"Plain"]]}]]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", RowBox[{ SubscriptBox[ StyleBox["I", FontSlant->"Plain"], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}]], "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], "1"], " ", SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], "2"], " ", RowBox[{"P", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}], RowBox[{ FractionBox[ RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontSlant->"Plain"]]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontSlant->"Plain"]]}]], ".", FractionBox[ RowBox[{\(\[PartialD]log\), " ", RowBox[{ SubscriptBox[ StyleBox["Q", FontSlant->"Plain"], "12"], "(", RowBox[{ StyleBox[ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], StyleBox[",", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox[ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}]}], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontSlant->"Plain"]]}]]}]}]}], SubscriptBox[ SubscriptBox["\[VerticalSeparator]", GridBox[{ { RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], "=", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ")"}]}]}, { RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "2"], "=", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "2"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}]}]} }]], \(\[IndentingNewLine]\)], "\n", "\[AlignmentMarker]", RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"]}]], "1"], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ")"}], FractionBox[ RowBox[{\(\[PartialD]log\), " ", RowBox[{\(Q\_1\), "(", RowBox[{ StyleBox[ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox[ SubscriptBox["s", StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontSlant->"Plain"]]}]]}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", RowBox[{ SubscriptBox[ StyleBox["I", FontSlant->"Plain"], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"]}]], "=", "\[AlignmentMarker]", "similar"}], TraditionalForm], "\n", FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", RowBox[{ SubscriptBox[ StyleBox["I", FontSlant->"Plain"], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}]], "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"]}]], "1"], " ", SubscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"]}]], "2"], " ", RowBox[{"P", "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], "2"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"]}], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}], FractionBox[ RowBox[{\(\[PartialD]log\), " ", RowBox[{\(Q\_12\), "(", RowBox[{ StyleBox[ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], StyleBox[",", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{ StyleBox[ SubscriptBox[ StyleBox["y", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}]}], ")"}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "12"]}]]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:29"], Cell[TextData[{ "The derivative of (the model of) the mutual information with respect to \ the parameters in the first stage of the network causes the training of the \ left and right cluster transformations to become mutually coupled [", ButtonBox["13", ButtonData:>"Ref:Luttrell1991c", ButtonStyle->"Hyperlink"], ", ", ButtonBox["14", ButtonData:>"Ref:Luttrell1992", ButtonStyle->"Hyperlink"], "]. This effect, which is called self-supervision, is important because it \ co-ordinates the training of different layers in a multilayer unsupervised \ network (of which the ACE network is but a particular example)." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". REFERENCES" }], "Section"], Cell[TextData[{ "1. R. T. Cox, \"Probability, frequency and reasonable expectation\", ", StyleBox["Am. J. Phys.", FontSlant->"Italic"], ", ", "vol. 14, pp. 1-13", ", 1946." }], "Reference", CellTags->"Ref:Cox1946"], Cell[TextData[{ "2. H. Jeffreys, ", StyleBox["Theory of Probability", FontSlant->"Italic"], ", Clarendon Press, 1939." }], "Reference", CellTags->"Ref:Jeffreys1939"], Cell[TextData[{ "3. D. H. Ackley, G. E. Hinton and T. J. Sejnowski, \"A learning algorithm \ for Boltzmann machines\", ", StyleBox["Cogn. Sci.", FontSlant->"Italic"], ", vol. 9, pp. 147-169, 1985." }], "Reference", CellTags->"Ref:AckleyHintonSejnowski1985"], Cell[TextData[{ "4. L. E. Baum, \"An inequality and associated maximisation technique in \ statistical estimation for probabilistic functions of Markov processes\", ", StyleBox["Inequalities", FontSlant->"Italic"], ", vol. 3, pp. 1-8, 1972." }], "Reference", CellTags->"Ref:Baum1972"], Cell[TextData[{ "5. A. P. Dempster, N. M. Laird and D. B. Rubin, \"Maximum likelihood from \ incomplete data via the EM algorithm\", ", StyleBox["J. Roy. Statist. Soc. Ser. B", FontSlant->"Italic"], ", vol. 39, pp. 1-37, 1977." }], "Reference", CellTags->"Ref:DempsterLairdRubin1977"], Cell[TextData[{ "6. R. Kindermann and J. L. Snell, \"Markov random fields and their \ applications\", ", StyleBox["Contemporary Mathematics", FontSlant->"Italic"], ", vol. 1, American Mathematical Society, 1980." }], "Reference", CellTags->"Ref:KindermannSnell1980"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/maxent88/maxent88.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "7. S. P. Luttrell, \"The use of Bayesian and entropic methods in neural \ network theory\", ", StyleBox["Maximum entropy and Bayesian methods", FontSlant->"Italic"], ", Skilling J (ed.), pp. 363-370, Kluwer Academic Publishers, Cambridge, \ 1989." }], "Reference", CellTags->"Ref:Luttrell1989a"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/4437/4437.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "8. S. P. Luttrell, \"A trainable texture anomaly detector using the \ Adaptive Cluster Expansion (ACE) method\", ", StyleBox["RSRE Technical Report no. 4437", FontSlant->"Italic"], ", 1990." }], "Reference", CellTags->"Ref:Luttrell1990"], Cell[TextData[{ "9. S. P. Luttrell S P, \"Adaptive Cluster Expansion (ACE): a multilayer \ network for estimating probability density functions\", submitted to ", StyleBox["IEEE Trans. NN", FontSlant->"Italic"], ", 1991." }], "Reference", CellTags->"Ref:Luttrell1991a"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/hiervq/hiervq.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "10. S. P. Luttrell, \"Hierarchical vector quantisation\", ", StyleBox["Proc. IEE Part I", FontSlant->"Italic"], ", vol. 136, pp. 405-413, 1989." }], "Reference", CellTags->"Ref:Luttrell1989b"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/compress/compress.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "11. S. P. Luttrell, \"Image compression using a multilayer neural \ network\", ", StyleBox["Patt. Recog. Lett.", FontSlant->"Italic"], ", vol. 10, pp. 1-7, 1989." }], "Reference", CellTags->"Ref:Luttrell1989c"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/spie91/spie91.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "12. S. P. Luttrell, \"A hierarchical network for clutter and texture \ modelling\", ", StyleBox["Proc. SPIE Conf. on Adaptive Signal Processing", FontSlant->"Italic"], ", Haykin S (ed.), pp. 518-528, San Diego, 1991." }], "Reference", CellTags->"Ref:Luttrell1991b"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/ieenn91/ieenn91.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "13. S. P. Luttrell, \"Self-supervised training of hierarchical vector \ quantisers\", ", StyleBox["Proc. 2nd IEE Conf. on Artificial Neural Networks", FontSlant->"Italic"], ", pp. 5-9, Bournemouth, 1991." }], "Reference", CellTags->"Ref:Luttrell1991c"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/selfsup/selfsup.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "14. S. P. Luttrell, \"Self-supervision in multilayer adaptive networks\", \ to appear in ", StyleBox["Proc. IEE Part F", FontSlant->"Italic"], ", 1992." }], "Reference", CellTags->"Ref:Luttrell1992"] }, Closed]] }, Open ]] }, FrontEndVersion->"5.0 for Microsoft Windows", ScreenRectangle->{{0, 1280}, {0, 941}}, WindowToolbars->{}, WindowSize->{665.375, 641}, WindowMargins->{{307.25, Automatic}, {Automatic, 50}}, Magnification->1, StyleDefinitions -> "Report.nb" ] (******************************************************************* Cached data follows. If you edit this Notebook file directly, not using Mathematica, you must remove the line containing CacheID at the top of the file. The cache data will then be recreated when you save this file from within Mathematica. *******************************************************************) (*CellTagsOutline CellTagsIndex->{ "Eq:1"->{ Cell[11604, 395, 3211, 82, 70, "NumberedEquation", CellTags->"Eq:1"]}, "Eq:2"->{ Cell[14964, 484, 1353, 41, 70, "NumberedEquation", CellTags->"Eq:2"]}, "Eq:3"->{ Cell[16832, 540, 3796, 98, 70, "NumberedEquation", CellTags->"Eq:3"]}, "Eq:4"->{ Cell[22524, 692, 2564, 72, 70, "NumberedEquation", CellTags->"Eq:4"]}, "Eq:5"->{ Cell[25290, 773, 5087, 126, 70, "NumberedEquation", CellTags->"Eq:5"]}, "Eq:6"->{ Cell[30888, 925, 2133, 61, 70, "NumberedEquation", CellTags->"Eq:6"]}, "Eq:7"->{ Cell[33660, 1012, 1621, 45, 70, "NumberedEquation", CellTags->"Eq:7"]}, "Eq:8"->{ Cell[36301, 1096, 1479, 45, 70, "NumberedEquation", CellTags->"Eq:8"]}, "Eq:9"->{ Cell[38772, 1185, 1504, 45, 70, "NumberedEquation", CellTags->"Eq:9"]}, "Eq:10"->{ Cell[40577, 1243, 2504, 76, 70, "NumberedEquation", CellTags->"Eq:10"]}, "Eq:11"->{ Cell[43640, 1340, 3975, 109, 70, "NumberedEquation", CellTags->"Eq:11"]}, "Fig:1"->{ Cell[47849, 1462, 271, 10, 70, "NumberedFigure", CellTags->"Fig:1"]}, "Eq:12"->{ Cell[48768, 1499, 2782, 80, 70, "NumberedEquation", CellTags->"Eq:12"]}, "Eq:13"->{ Cell[51865, 1591, 3378, 90, 70, "NumberedEquation", CellTags->"Eq:13"]}, "Fig:2"->{ Cell[55805, 1705, 271, 10, 70, "NumberedFigure", CellTags->"Fig:2"]}, "Eq:14"->{ Cell[56936, 1747, 2187, 64, 70, "NumberedEquation", CellTags->"Eq:14"]}, "Eq:15"->{ Cell[59696, 1829, 3734, 105, 70, "NumberedEquation", CellTags->"Eq:15"]}, "Eq:16"->{ Cell[65240, 2002, 2589, 74, 70, "NumberedEquation", CellTags->"Eq:16"]}, "Eq:17"->{ Cell[68038, 2087, 3932, 110, 70, "NumberedEquation", CellTags->"Eq:17"]}, "Eq:18"->{ Cell[72644, 2217, 4232, 119, 70, "NumberedEquation", CellTags->"Eq:18"]}, "Eq:19"->{ Cell[77070, 2344, 3480, 98, 70, "NumberedEquation", CellTags->"Eq:19"]}, "Eq:20"->{ Cell[81688, 2478, 10370, 260, 70, "NumberedEquation", CellTags->"Eq:20"]}, "Eq:21"->{ Cell[92278, 2749, 10301, 253, 70, "NumberedEquation", CellTags->"Eq:21"]}, "Eq:22"->{ Cell[103012, 3019, 9976, 265, 70, "NumberedEquation", CellTags->"Eq:22"]}, "Eq:23"->{ Cell[113131, 3294, 2028, 62, 70, "NumberedEquation", CellTags->"Eq:23"]}, "Eq:24"->{ Cell[115847, 3383, 4591, 132, 70, "NumberedEquation", CellTags->"Eq:24"]}, "Eq:25"->{ Cell[120745, 3524, 3852, 101, 70, "NumberedEquation", CellTags->"Eq:25"]}, "Fig:3"->{ Cell[125178, 3646, 271, 10, 70, "NumberedFigure", CellTags->"Fig:3"]}, "Eq:26"->{ Cell[126604, 3702, 4116, 123, 70, "NumberedEquation", CellTags->"Eq:26"]}, "Eq:27"->{ Cell[130970, 3837, 2197, 65, 70, "NumberedEquation", CellTags->"Eq:27"]}, "Eq:28"->{ Cell[133448, 3914, 2790, 75, 70, "NumberedEquation", CellTags->"Eq:28"]}, "Fig:4"->{ Cell[136484, 4002, 271, 10, 70, "NumberedFigure", CellTags->"Fig:4"]}, "Ed:Change1"->{ Cell[138279, 4069, 7347, 190, 70, "Text", CellTags->"Ed:Change1"]}, "Eq:29"->{ Cell[145887, 4275, 16355, 420, 70, "NumberedEquation", CellTags->"Eq:29"]}, "Ref:Cox1946"->{ Cell[163024, 4723, 230, 8, 70, "Reference", CellTags->"Ref:Cox1946"]}, "Ref:Jeffreys1939"->{ Cell[163257, 4733, 177, 6, 70, "Reference", CellTags->"Ref:Jeffreys1939"]}, "Ref:AckleyHintonSejnowski1985"->{ Cell[163437, 4741, 269, 7, 70, "Reference", CellTags->"Ref:AckleyHintonSejnowski1985"]}, "Ref:Baum1972"->{ Cell[163709, 4750, 296, 7, 70, "Reference", CellTags->"Ref:Baum1972"]}, "Ref:DempsterLairdRubin1977"->{ Cell[164008, 4759, 296, 7, 70, "Reference", CellTags->"Ref:DempsterLairdRubin1977"]}, "Ref:KindermannSnell1980"->{ Cell[164307, 4768, 278, 7, 70, "Reference", CellTags->"Ref:KindermannSnell1980"]}, "Ref:Luttrell1989a"->{ Cell[164588, 4777, 547, 18, 70, "Reference", CellTags->"Ref:Luttrell1989a"]}, "Ref:Luttrell1990"->{ Cell[165138, 4797, 474, 16, 70, "Reference", CellTags->"Ref:Luttrell1990"]}, "Ref:Luttrell1991a"->{ Cell[165615, 4815, 281, 7, 70, "Reference", CellTags->"Ref:Luttrell1991a"]}, "Ref:Luttrell1989b"->{ Cell[165899, 4824, 433, 15, 70, "Reference", CellTags->"Ref:Luttrell1989b"]}, "Ref:Luttrell1989c"->{ Cell[166335, 4841, 462, 17, 70, "Reference", CellTags->"Ref:Luttrell1989c"]}, "Ref:Luttrell1991b"->{ Cell[166800, 4860, 506, 16, 70, "Reference", CellTags->"Ref:Luttrell1991b"]}, "Ref:Luttrell1991c"->{ Cell[167309, 4878, 495, 16, 70, "Reference", CellTags->"Ref:Luttrell1991c"]}, "Ref:Luttrell1992"->{ Cell[167807, 4896, 442, 16, 70, "Reference", CellTags->"Ref:Luttrell1992"]} } *) (*CellTagsIndex CellTagsIndex->{ {"Eq:1", 168946, 4934}, {"Eq:2", 169039, 4937}, {"Eq:3", 169132, 4940}, {"Eq:4", 169225, 4943}, {"Eq:5", 169318, 4946}, {"Eq:6", 169412, 4949}, {"Eq:7", 169505, 4952}, {"Eq:8", 169599, 4955}, {"Eq:9", 169693, 4958}, {"Eq:10", 169788, 4961}, {"Eq:11", 169884, 4964}, {"Fig:1", 169981, 4967}, {"Eq:12", 170074, 4970}, {"Eq:13", 170170, 4973}, {"Fig:2", 170266, 4976}, {"Eq:14", 170359, 4979}, {"Eq:15", 170455, 4982}, {"Eq:16", 170552, 4985}, {"Eq:17", 170648, 4988}, {"Eq:18", 170745, 4991}, {"Eq:19", 170842, 4994}, {"Eq:20", 170938, 4997}, {"Eq:21", 171036, 5000}, {"Eq:22", 171134, 5003}, {"Eq:23", 171232, 5006}, {"Eq:24", 171329, 5009}, {"Eq:25", 171427, 5012}, {"Fig:3", 171525, 5015}, {"Eq:26", 171619, 5018}, {"Eq:27", 171717, 5021}, {"Eq:28", 171814, 5024}, {"Fig:4", 171911, 5027}, {"Ed:Change1", 172010, 5030}, {"Eq:29", 172101, 5033}, {"Ref:Cox1946", 172206, 5036}, {"Ref:Jeffreys1939", 172311, 5039}, {"Ref:AckleyHintonSejnowski1985", 172434, 5042}, {"Ref:Baum1972", 172553, 5045}, {"Ref:DempsterLairdRubin1977", 172669, 5048}, {"Ref:KindermannSnell1980", 172796, 5051}, {"Ref:Luttrell1989a", 172914, 5054}, {"Ref:Luttrell1990", 173026, 5057}, {"Ref:Luttrell1991a", 173138, 5060}, {"Ref:Luttrell1989b", 173250, 5063}, {"Ref:Luttrell1989c", 173363, 5066}, {"Ref:Luttrell1991b", 173476, 5069}, {"Ref:Luttrell1991c", 173589, 5072}, {"Ref:Luttrell1992", 173701, 5075} } *) (*NotebookFileOutline Notebook[{ Cell[CellGroupData[{ Cell[1776, 53, 26, 0, 70, "Section 1"], Cell[CellGroupData[{ Cell[1827, 57, 39, 0, 70, "Subsection"], Cell[1869, 59, 211, 8, 70, "Text"], Cell[2083, 69, 370, 6, 70, "Text"], Cell[2456, 77, 407, 14, 70, "Text"], Cell[2866, 93, 720, 26, 70, "Text"], Cell[3589, 121, 1911, 62, 70, "Text"], Cell[5503, 185, 2591, 85, 70, "Text"], Cell[8097, 272, 262, 8, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[8408, 286, 43, 0, 70, "Title"], Cell[8454, 288, 178, 6, 70, "Author"], Cell[8635, 296, 135, 3, 70, "Text"], Cell[8773, 301, 301, 7, 70, "Abstract"], Cell[CellGroupData[{ Cell[9099, 312, 79, 4, 70, "Section 1"], Cell[9181, 318, 837, 19, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[10055, 342, 94, 4, 70, "Section"], Cell[10152, 348, 1008, 27, 70, "Text"], Cell[CellGroupData[{ Cell[11185, 379, 136, 7, 70, "Subsection"], Cell[11324, 388, 277, 5, 70, "Text"], Cell[11604, 395, 3211, 82, 70, "NumberedEquation", CellTags->"Eq:1"], Cell[14818, 479, 143, 3, 70, "Text"], Cell[14964, 484, 1353, 41, 70, "NumberedEquation", CellTags->"Eq:2"], Cell[16320, 527, 328, 5, 70, "Text"], Cell[16651, 534, 178, 4, 70, "Text"], Cell[16832, 540, 3796, 98, 70, "NumberedEquation", CellTags->"Eq:3"], Cell[20631, 640, 698, 16, 70, "Text"], Cell[21332, 658, 729, 13, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[22098, 676, 133, 7, 70, "Subsection"], Cell[22234, 685, 287, 5, 70, "Text"], Cell[22524, 692, 2564, 72, 70, "NumberedEquation", CellTags->"Eq:4"], Cell[25091, 766, 141, 3, 70, "Text"], Cell[25235, 771, 52, 0, 70, "Text"], Cell[25290, 773, 5087, 126, 70, "NumberedEquation", CellTags->"Eq:5"], Cell[30380, 901, 60, 0, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[30477, 906, 148, 7, 70, "Subsection"], Cell[30628, 915, 257, 8, 70, "Text"], Cell[30888, 925, 2133, 61, 70, "NumberedEquation", CellTags->"Eq:6"], Cell[33024, 988, 309, 8, 70, "Text"], Cell[33336, 998, 321, 12, 70, "Text"], Cell[33660, 1012, 1621, 45, 70, "NumberedEquation", CellTags->"Eq:7"], Cell[35284, 1059, 957, 33, 70, "Text"], Cell[36244, 1094, 54, 0, 70, "Text"], Cell[36301, 1096, 1479, 45, 70, "NumberedEquation", CellTags->"Eq:8"], Cell[37783, 1143, 473, 10, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[38305, 1159, 97, 4, 70, "Section"], Cell[38405, 1165, 114, 3, 70, "Text"], Cell[CellGroupData[{ Cell[38544, 1172, 125, 7, 70, "Subsection"], Cell[38672, 1181, 97, 2, 70, "Text"], Cell[38772, 1185, 1504, 45, 70, "NumberedEquation", CellTags->"Eq:9"], Cell[40279, 1232, 295, 9, 70, "Text"], Cell[40577, 1243, 2504, 76, 70, "NumberedEquation", CellTags->"Eq:10"], Cell[43084, 1321, 553, 17, 70, "Text"], Cell[43640, 1340, 3975, 109, 70, "NumberedEquation", CellTags->"Eq:11"], Cell[47618, 1451, 228, 9, 70, "Text"], Cell[47849, 1462, 271, 10, 70, "NumberedFigure", CellTags->"Fig:1"], Cell[48123, 1474, 277, 5, 70, "Caption"] }, Closed]], Cell[CellGroupData[{ Cell[48437, 1484, 124, 7, 70, "Subsection"], Cell[48564, 1493, 201, 4, 70, "Text"], Cell[48768, 1499, 2782, 80, 70, "NumberedEquation", CellTags->"Eq:12"], Cell[51553, 1581, 309, 8, 70, "Text"], Cell[51865, 1591, 3378, 90, 70, "NumberedEquation", CellTags->"Eq:13"], Cell[55246, 1683, 326, 9, 70, "Text"], Cell[55575, 1694, 227, 9, 70, "Text"], Cell[55805, 1705, 271, 10, 70, "NumberedFigure", CellTags->"Fig:2"], Cell[56079, 1717, 440, 7, 70, "Caption"] }, Closed]], Cell[CellGroupData[{ Cell[56556, 1729, 122, 7, 70, "Subsection"], Cell[56681, 1738, 252, 7, 70, "Text"], Cell[56936, 1747, 2187, 64, 70, "NumberedEquation", CellTags->"Eq:14"], Cell[59126, 1813, 141, 3, 70, "Text"], Cell[59270, 1818, 423, 9, 70, "Text"], Cell[59696, 1829, 3734, 105, 70, "NumberedEquation", CellTags->"Eq:15"], Cell[63433, 1936, 361, 8, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[63843, 1950, 97, 4, 70, "Section"], Cell[63943, 1956, 865, 26, 70, "Text"], Cell[CellGroupData[{ Cell[64833, 1986, 119, 7, 70, "Subsection"], Cell[64955, 1995, 282, 5, 70, "Text"], Cell[65240, 2002, 2589, 74, 70, "NumberedEquation", CellTags->"Eq:16"], Cell[67832, 2078, 111, 3, 70, "Text"], Cell[67946, 2083, 89, 2, 70, "Text"], Cell[68038, 2087, 3932, 110, 70, "NumberedEquation", CellTags->"Eq:17"], Cell[71973, 2199, 668, 16, 70, "Text"], Cell[72644, 2217, 4232, 119, 70, "NumberedEquation", CellTags->"Eq:18"], Cell[76879, 2338, 188, 4, 70, "Text"], Cell[77070, 2344, 3480, 98, 70, "NumberedEquation", CellTags->"Eq:19"], Cell[80553, 2444, 659, 10, 70, "Text"], Cell[81215, 2456, 185, 4, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[81437, 2465, 158, 7, 70, "Subsection"], Cell[81598, 2474, 87, 2, 70, "Text"], Cell[81688, 2478, 10370, 260, 70, "NumberedEquation", CellTags->"Eq:20"], Cell[92061, 2740, 214, 7, 70, "Text"], Cell[92278, 2749, 10301, 253, 70, "NumberedEquation", CellTags->"Eq:21"], Cell[102582, 3004, 284, 8, 70, "Text"], Cell[102869, 3014, 140, 3, 70, "Text"], Cell[103012, 3019, 9976, 265, 70, "NumberedEquation", CellTags->"Eq:22"], Cell[112991, 3286, 137, 6, 70, "Text"], Cell[113131, 3294, 2028, 62, 70, "NumberedEquation", CellTags->"Eq:23"], Cell[115162, 3358, 296, 5, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[115495, 3368, 129, 7, 70, "Subsection"], Cell[115627, 3377, 217, 4, 70, "Text"], Cell[115847, 3383, 4591, 132, 70, "NumberedEquation", CellTags->"Eq:24"], Cell[120441, 3517, 301, 5, 70, "Text"], Cell[120745, 3524, 3852, 101, 70, "NumberedEquation", CellTags->"Eq:25"], Cell[124600, 3627, 317, 5, 70, "Text"], Cell[124920, 3634, 255, 10, 70, "Text"], Cell[125178, 3646, 271, 10, 70, "NumberedFigure", CellTags->"Fig:3"], Cell[125452, 3658, 169, 3, 70, "Caption"], Cell[125624, 3663, 167, 3, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[125828, 3671, 122, 7, 70, "Subsection"], Cell[125953, 3680, 520, 15, 70, "Text"], Cell[126476, 3697, 125, 3, 70, "Text"], Cell[126604, 3702, 4116, 123, 70, "NumberedEquation", CellTags->"Eq:26"], Cell[130723, 3827, 244, 8, 70, "Text"], Cell[130970, 3837, 2197, 65, 70, "NumberedEquation", CellTags->"Eq:27"], Cell[133170, 3904, 275, 8, 70, "Text"], Cell[133448, 3914, 2790, 75, 70, "NumberedEquation", CellTags->"Eq:28"], Cell[136241, 3991, 240, 9, 70, "Text"], Cell[136484, 4002, 271, 10, 70, "NumberedFigure", CellTags->"Fig:4"], Cell[136758, 4014, 358, 6, 70, "Caption"], Cell[137119, 4022, 187, 4, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[137355, 4032, 76, 4, 70, "Section"], Cell[137434, 4038, 588, 9, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[138059, 4052, 73, 4, 70, "Section"], Cell[CellGroupData[{ Cell[138157, 4060, 119, 7, 70, "Subsection"], Cell[138279, 4069, 7347, 190, 70, "Text", CellTags->"Ed:Change1"] }, Closed]], Cell[CellGroupData[{ Cell[145663, 4264, 161, 7, 70, "Subsection"], Cell[145827, 4273, 57, 0, 70, "Text"], Cell[145887, 4275, 16355, 420, 70, "NumberedEquation", CellTags->"Eq:29"], Cell[162245, 4697, 652, 14, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[162946, 4717, 75, 4, 70, "Section"], Cell[163024, 4723, 230, 8, 70, "Reference", CellTags->"Ref:Cox1946"], Cell[163257, 4733, 177, 6, 70, "Reference", CellTags->"Ref:Jeffreys1939"], Cell[163437, 4741, 269, 7, 70, "Reference", CellTags->"Ref:AckleyHintonSejnowski1985"], Cell[163709, 4750, 296, 7, 70, "Reference", CellTags->"Ref:Baum1972"], Cell[164008, 4759, 296, 7, 70, "Reference", CellTags->"Ref:DempsterLairdRubin1977"], Cell[164307, 4768, 278, 7, 70, "Reference", CellTags->"Ref:KindermannSnell1980"], Cell[164588, 4777, 547, 18, 70, "Reference", CellTags->"Ref:Luttrell1989a"], Cell[165138, 4797, 474, 16, 70, "Reference", CellTags->"Ref:Luttrell1990"], Cell[165615, 4815, 281, 7, 70, "Reference", CellTags->"Ref:Luttrell1991a"], Cell[165899, 4824, 433, 15, 70, "Reference", CellTags->"Ref:Luttrell1989b"], Cell[166335, 4841, 462, 17, 70, "Reference", CellTags->"Ref:Luttrell1989c"], Cell[166800, 4860, 506, 16, 70, "Reference", CellTags->"Ref:Luttrell1991b"], Cell[167309, 4878, 495, 16, 70, "Reference", CellTags->"Ref:Luttrell1991c"], Cell[167807, 4896, 442, 16, 70, "Reference", CellTags->"Ref:Luttrell1992"] }, Closed]] }, Open ]] } ] *) (******************************************************************* End of Mathematica Notebook file. *******************************************************************)