(************** Content-type: application/mathematica ************** CreatedBy='Mathematica 5.0' Mathematica-Compatible Notebook This notebook can be used with any Mathematica-compatible application, such as Mathematica, MathReader or Publicon. The data for the notebook starts with the line containing stars above. To get the notebook into a Mathematica-compatible application, do one of the following: * Save the data starting with the line of stars above into a file with a name ending in .nb, then open the file inside the application; * Copy the data starting with the line of stars above to the clipboard, then use the Paste menu command inside the application. Data for notebooks contains only printable 7-bit ASCII and can be sent directly in email or through ftp in text mode. Newlines can be CR, LF or CRLF (Unix, Macintosh or MS-DOS style). NOTE: If you modify the data for this notebook not in a Mathematica- compatible application, you must delete the line below containing the word CacheID, otherwise Mathematica-compatible applications may try to use invalid cache data. For more information on notebooks and Mathematica-compatible applications, contact Wolfram Research: web: http://www.wolfram.com email: info@wolfram.com phone: +1-217-398-0700 (U.S.) Notebook reader applications are available free of charge from Wolfram Research. *******************************************************************) (*CacheID: 232*) (*NotebookFileLineBreakTest NotebookFileLineBreakTest*) (*NotebookOptionsPosition[ 132924, 4448]*) (*NotebookOutlinePosition[ 141610, 4705]*) (* CellTagsIndexPosition[ 139670, 4643]*) (*WindowFrame->Normal*) Notebook[{ Cell[CellGroupData[{ Cell["Notes", "Section 1"], Cell[CellGroupData[{ Cell["Editorial Changes", "Subsection"], Cell[TextData[{ "References of the form \"", StyleBox["v", FontWeight->"Bold"], "(", StyleBox["n", FontWeight->"Bold"], ")\" changed to \"", StyleBox["v", FontWeight->"Bold"], "(n)\" throughout the references section." }], "Text"], Cell["\<\ Equations reformatted so that one equation appears per line throughout the \ paper.\ \>", "Text"], Cell[TextData[{ "\"", Cell[BoxData[ FormBox[ SubscriptBox["\[Del]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["\[Del]", FontWeight->"Bold"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]], TraditionalForm]]], "\" throughout the paper." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change1", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"Cogn. Sci.\" changed to \"", StyleBox["Cogn. Sci.", FontSlant->"Italic"], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change2", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"Random\" changed to \"random\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change3", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"Metropolis N, Rosenbluth A W, Rosenbluth M N and Teller A H\" changed to \ \"Metropolis N, Rosenbluth A W, Rosenbluth M N, Teller A H and Teller E\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change4", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"sections 2.1 and 2.2\" changed to \"section 2.1 and section 2.2\" to \ accommodate two hyperlinks." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change5", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"exp", "[", RowBox[{ StyleBox["k", FontWeight->"Bold", FontSlant->"Plain"], ".", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}], "]"}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{"exp", "[", RowBox[{"-", RowBox[{ StyleBox["k", FontWeight->"Bold", FontSlant->"Plain"], ".", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}], "]"}], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change6", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"ie\" changed to \"i.e.\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change7", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"\[Delta]H", "/", RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox["\[Delta]H", RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], TraditionalForm]]], "\" and \"", Cell[BoxData[ FormBox[ RowBox[{"\[Delta]", RowBox[{ RowBox[{"\[LeftAngleBracket]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[RightAngleBracket]"}], "/", RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{"\[Delta]", RowBox[{"\[LeftAngleBracket]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[RightAngleBracket]"}]}], RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change8", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"\[PartialD]", StyleBox["G", FontSlant->"Plain"]}], StyleBox["/", FontSlant->"Plain"], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["k", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{"\[PartialD]", StyleBox["G", FontSlant->"Plain"]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["k", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]]}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change9", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"\[Delta]G", "/", RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox["\[Delta]G", RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["ERROR", ButtonData:>"Ed:Change6", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"A particularly desirable property of this learning algorithm is that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " tends to become constant, thus maximising the output entropy. \ Furthermore, the algorithm can be shown to minimise ", Cell[BoxData[ FormBox[ RowBox[{"<", RowBox[{"log", " ", RowBox[{"(", RowBox[{ StyleBox["V", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], ")"}]}], " ", ">"}], TraditionalForm]]], " where ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["V", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is the error volume associated with the reconstruction of ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " from ", Cell[BoxData[ FormBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " using ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " [", ButtonBox["Luttrell, 1988b", ButtonData:>"Ref:Luttrell1988b", ButtonStyle->"Hyperlink"], "].\" is incorrect." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["\<\ THE USE OF BAYESIAN AND ENTROPIC METHODS IN NEURAL NETWORK THEORY\ \>", "Title"], Cell["\<\ S. P. Luttrell Royal Signals and Radar Establishment St. Andrews Road Malvern WORCS. WR14 SPS U.K.\ \>", "Author"], Cell["\<\ This appeared in Maximum Entropy and Bayesian Methods, Kluwer, J. Skilling \ (ed.), 363-370, 1989.\ \>", "Text"], Cell["\[Copyright] 1989 Controller, HMSO, London.", "Text"], Cell["\<\ ABSTRACT. There has been much interest recently in the use of neural networks \ to solve complicated information processing problems such as those which \ arise in signal and image processing. In this paper we review Markov random \ field (MRF) neural network techniques for representing joint probability \ density functions (PDF). The \"Boltzmann machine\" serves as the paradigm, \ and we present a generalised version of its learning algorithm. We also \ present a technique for designing MRF potentials with low information \ redundancy for modelling image texture. To improve further the computational \ efficiency of such neural networks we introduce a novel method of cluster \ decomposing a PDF by using topographic mappings. The outcome of this \ programme is a means of designing sampling functions for extracting \ information from datasets (typically images).\ \>", "Abstract"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". Introduction" }], "Section 1"], Cell[TextData[{ "The image processing community has shown much interest in the use of \ Maxkov random field models to describe probability density functions for use \ in Bayesian image reconstruction schemes [", ButtonBox["Geman and Geman 1984", ButtonData:>"Ref:GemanGeman1984", ButtonStyle->"Hyperlink"], ", ", ButtonBox["Geman and Graffigne 1987", ButtonData:>"Ref:GemanGraffigne1987", ButtonStyle->"Hyperlink"], "]. If we denote the fileld state as ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " and the PDF over states as ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " then an MRF is defined by a consistent set of conditional PDFs (called \ characteristics) amongst the components of ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ". It follows from the Hammersley-Clifford theorem that corresponding to \ each consistently defined MRF there is an equivalent Gibbs distribution [", ButtonBox["Besag 1974", ButtonData:>"Ref:Besag1974", ButtonStyle->"Hyperlink"], ", ", ButtonBox["Kindermann and Snell 1980", ButtonData:>"Ref:KindermannSnell1980", ButtonStyle->"Hyperlink"], ", ", ButtonBox["Preston 1974", ButtonData:>"Ref:Preston1974", ButtonStyle->"Hyperlink"], "], so ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " may be written as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "=", RowBox[{ FractionBox["1", StyleBox["Z", FontSlant->"Plain"]], RowBox[{"exp", "[", RowBox[{"-", RowBox[{ StyleBox["k", FontWeight->"Bold", FontSlant->"Plain"], ".", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}], "]"}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:1"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is a vector potential, ", Cell[BoxData[ FormBox[ StyleBox["k", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " is a vector of coefficients, and ", Cell[BoxData[ FormBox[ StyleBox["Z", FontSlant->"Plain"], TraditionalForm]]], " is a partition function. We use an unconventional symbol ", Cell[BoxData[ FormBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " to denote the potential because it is in fact a set of sampling functions \ of ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ". ", ButtonBox["Equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:1"], ") defines a ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " from which samples ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " may be drawn by using a Monte Carlo scheme such as the Metropolis \ algorithm [", ButtonBox["Metropolis et al, 1953", ButtonData:>"Ref:MetropolisRosenbluthRosenbluthTellerTeller1953", ButtonStyle->"Hyperlink"], "] or some variant thereof." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is, of course, the maximum entropy PDF (with a uniform prior) which is \ consistent with the set of constraints ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"\[LeftAngleBracket]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[RightAngleBracket]"}], "=", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "0"]}], TraditionalForm]]], ", where ", Cell[BoxData[ \(TraditionalForm\`\[LeftAngleBracket]\[CenterEllipsis]\ \[RightAngleBracket]\)]], " denotes an average over ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " [", ButtonBox["Jaynes 1957", ButtonData:>"Ref:Jaynes1957a", ButtonStyle->"Hyperlink"], ", ", ButtonBox["1968", ButtonData:>"Ref:Jaynes1968", ButtonStyle->"Hyperlink"], ", ", ButtonBox["1982", ButtonData:>"Ref:Jaynes1982", ButtonStyle->"Hyperlink"], "]. Note that ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " has the form given in ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:1"], ") if, and only if, the functional derivative ", Cell[BoxData[ FormBox[ FractionBox["\[Delta]H", RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], TraditionalForm]]], " lies in the function subspace spanned by the vector of functional \ derivatives ", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{"\[Delta]", RowBox[{"\[LeftAngleBracket]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[RightAngleBracket]"}]}], RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["P", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], TraditionalForm]]], ", where ", Cell[BoxData[ FormBox[ StyleBox["H", FontSlant->"Plain"], TraditionalForm]]], " denotes the entropy of ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], "." }], "Text", CellTags->"Ed:Change7"], Cell[TextData[{ "The purpose of this paper is to extend the above MRF scheme by introducing \ a greater degree of adaptability into the model. Thus in ", ButtonBox["section", ButtonData:>"Sect:2.1", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2.1"], ".", CounterBox["Subsection", "Sect:2.1"], " we shall explain how the Boltzmann machine neural network (and its \ generalisations) can be used to leaxn MRF models adaptively, and in ", ButtonBox["section", ButtonData:>"Sect:2.2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2.2"], ".", CounterBox["Subsection", "Sect:2.2"], " we shall explain how economical MRF models of image texture can be \ constructed. In ", ButtonBox["section", ButtonData:>"Sect:3.1", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:3.1"], ".", CounterBox["Subsection", "Sect:3.1"], " we shall introduce a novel form of multilayer neural network which allows \ maximum entropy reconstructions of the input PDF to be constructed with \ minimal computational effort, and in ", ButtonBox["section", ButtonData:>"Sect:3.2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:3.2"], ".", CounterBox["Subsection", "Sect:3.2"], " we shall explain how topographic mappings can be used to implement the \ layer to layer transformations in such a network." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], "-maximisation models" }], "Section", CellTags->"Sect:2"], Cell[TextData[{ ButtonBox["Equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:1"], ") is inflexible because ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " must be selected by hand: there is no means of deriving ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " adaptively from a training set of samples ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " following some observed PDF ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " (", Cell[BoxData[ FormBox[ RowBox[{"\[NotEqual]", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], TraditionalForm]]], " in general). In order to acquire ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " adaptively we need a measure of the similarity of the (true) observed PDF \ ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " and the (maximum entropy) hypothesised PDF ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " defined in ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:1"], "). Define the relative entropy ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]] }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ StyleBox["G", FontSlant->"Plain"], "\[Congruent]", RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{"log", "[", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], "]"}]}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:2"], Cell[TextData[{ "Assuming base 2 logarithms, ", Cell[BoxData[ \(TraditionalForm\`2\^nG\)]], " is the probability that the hypothesised ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " will generate high probability ", Cell[BoxData[ FormBox[ StyleBox["n", FontSlant->"Plain"], TraditionalForm]], FontSlant->"Italic"], "-sample sequences of states ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " which belong to the high probability set generated by the true ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ", where ", Cell[BoxData[ FormBox[ StyleBox["n", FontSlant->"Plain"], TraditionalForm]]], " is asympotically laxge. Note that ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["G", FontSlant->"Plain"], "\[LessEqual]", "0"}], TraditionalForm]]], ", with ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["G", FontSlant->"Plain"], "=", RowBox[{ RowBox[{"0", " ", "iff", " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "=", RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}], TraditionalForm]]], ". We shall deal with two types of adaptation in ", ButtonBox["section", ButtonData:>"Sect:2.1", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2.1"], ".", CounterBox["Subsection", "Sect:2.1"], " and ", ButtonBox["section", ButtonData:>"Sect:2.2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2.2"], ".", CounterBox["Subsection", "Sect:2.2"], "." }], "Text", CellTags->"Ed:Change4"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " THE BOLTZMANN MACHINE" }], "Subsection", CellTags->"Sect:2.1"], Cell[TextData[{ "For a fixed set of potentials ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " we may optimise ", Cell[BoxData[ FormBox[ StyleBox["k", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " by ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], "-maximisation using" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", StyleBox["G", FontSlant->"Plain"]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["k", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]]}]], "=", RowBox[{ RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], FractionBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], FractionBox[ RowBox[{"\[PartialD]", RowBox[{ StyleBox["P", FontSlant->"Plain"], StyleBox["(", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[")", FontSlant->"Plain"]}]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["k", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]]}]]}]}], "=", RowBox[{ SubscriptBox[ RowBox[{"\[LeftAngleBracket]", RowBox[{ SubscriptBox[ StyleBox["s", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[RightAngleBracket]"}], RowBox[{ StyleBox["P", FontSlant->"Plain"], StyleBox["(", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[")", FontSlant->"Plain"]}]], "-", SubscriptBox[ RowBox[{"\[LeftAngleBracket]", RowBox[{ SubscriptBox[ StyleBox["s", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[RightAngleBracket]"}], RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[")", FontSlant->"Plain"]}]]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:3"], Cell[TextData[{ Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", StyleBox["G", FontSlant->"Plain"]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["k", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]]}]], "=", "0"}], TraditionalForm]]], " when the constraints ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"\[LeftAngleBracket]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[RightAngleBracket]"}], "=", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "0"]}], TraditionalForm]]], " are satisfied, so hill-climbing ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], " in ", Cell[BoxData[ FormBox[ StyleBox["k", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], "-space yields the required maximum entropy PDF estimate. The first term in \ ", ButtonBox["equation", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:3"], ") is estimated from Monte Carlo samples of ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " defined in ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:1"], "), whereas the second term is estimated from the training set which \ implicitly defines ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], "." }], "Text", CellTags->"Ed:Change8"], Cell[TextData[{ "We may construct a version of ", ButtonBox["equation", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:3"], ") in which ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " itself is effectively learnt at the same time as the coefficient vector \ ", Cell[BoxData[ FormBox[ StyleBox["k", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ". Thus we introduce hidden variables ", Cell[BoxData[ FormBox[ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " by augmenting ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " to become ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], ". Denoting the associated Gibbs distribution as ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], " leads to" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "=", RowBox[{ FractionBox["1", StyleBox["Z", FontSlant->"Plain"]], RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{"exp", "[", RowBox[{"-", RowBox[{ StyleBox["k", FontWeight->"Bold", FontSlant->"Plain"], ".", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}]}], "]"}]}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->{"Eq:4", "Ed:Change5"}], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ StyleBox["Z", FontSlant->"Plain"], TraditionalForm]]], " is now the partition function over all states ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], ". Defining ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], " as in ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:1"], ") then leads to" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ RowBox[{"\[PartialD]", StyleBox["G", FontSlant->"Plain"]}], RowBox[{"\[PartialD]", SubscriptBox[ StyleBox["k", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]]}]], "=", RowBox[{ SubscriptBox[ RowBox[{"\[LeftAngleBracket]", RowBox[{ SubscriptBox[ StyleBox["s", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "\[RightAngleBracket]"}], RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]], "-", SubscriptBox[ RowBox[{"\[LeftAngleBracket]", RowBox[{ SubscriptBox[ StyleBox["s", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "\[RightAngleBracket]"}], RowBox[{ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", RowBox[{ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[")", FontSlant->"Plain"]}]}]]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:5"], Cell[TextData[{ "The first term in ", ButtonBox["equation", ButtonData:>"Eq:5", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:5"], ") is estimated by using a Monte Carlo procedure, whereas the second term \ is a hybrid which uses the training set to provide samples from ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[")", FontSlant->"Plain"]}], TraditionalForm]]], " and a Monte Carlo procedure (with ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " held constant) to provide samples from ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", RowBox[{ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], ". The advantage of introducing the hidden variables ", Cell[BoxData[ FormBox[ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " is that a complicated ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox[")", FontSlant->"Plain"]}], TraditionalForm]]], " can be generated by using simple ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], " because the effect of ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "-", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], "-", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], TraditionalForm]]], " interactions \"dresses\" the bare ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], TraditionalForm]]], " interactions. This amounts to learning ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " adaptively by adjusting the strengths of the interactions with and \ amongst the hidden variables." }], "Text"], Cell[TextData[{ "The so-called Boltzmann machine [", ButtonBox["Ackley et al, 1985", ButtonData:>"Ref:AckleyHintonSejnowski1985", ButtonStyle->"Hyperlink"], "] is a simple form of hidden variable model which uses binary variables ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " and quadratic interactions ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["h", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], " together with ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], "-maximisation. More general hidden variable models have been discussed \ elsewhere [", ButtonBox["Luttrell, 1985", ButtonData:>"Ref:Luttrell1985", ButtonStyle->"Hyperlink"], "; ", ButtonBox["Sejnowski, 1986", ButtonData:>"Ref:Sejnowski1986", ButtonStyle->"Hyperlink"], "]. Whilst the Boltzmann machine is very flexible in its ability to adapt \ to the statistical properties of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ", it is computationally very inefficient due to the extensive Monte Carlo \ simulations which are required." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " DESIGNING POTENTIALS" }], "Subsection", CellTags->"Sect:2.2"], Cell[TextData[{ "There is another ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], "-maximisation approach to learning ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " for which the constraints are not on ", Cell[BoxData[ FormBox[ RowBox[{"\[LeftAngleBracket]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[RightAngleBracket]"}], TraditionalForm]]], " but on the whole PDF ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ". In general ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is given by" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[Congruent]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:6"], Cell[TextData[{ "where the Dirac delta function constrains the ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " integral as required. If the ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " dependence of the observed ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " can be expressed entirely in terms of ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ", then ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is a sufficient set of statistics [", ButtonBox["DeGroot, 1970", ButtonData:>"Ref:DeGroot1970", ButtonStyle->"Hyperlink"], "; ", ButtonBox["Luttrell, 1987a", ButtonData:>"Ref:Luttrell1987a", ButtonStyle->"Hyperlink"], "]. The maximum entropy reconstruction (with a uniform prior) will then be \ ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " if the entire PDF ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is used as a constraint. On the other hand, when ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is not a sufficient set of statistics, the maximum entropy method will, \ as usual, give the least committal reconstruction ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " which is consistent with ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " [", ButtonBox["Luttrell, 1988a", ButtonData:>"Ref:Luttrell1988a", ButtonStyle->"Hyperlink"], "]" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "=", "\[AlignmentMarker]", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], ")"}], RowBox[{ StyleBox["Z", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], ")"}]]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ StyleBox["Z", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[Congruent]", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{"\[Delta]", "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:7"], Cell[TextData[{ Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Z", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is proportional to the number of states ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " which map to ", Cell[BoxData[ FormBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ": it plays the role of a normalisation factor (it is not a partition \ function). With this interpretation the expression for ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:7", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:7"], ") is intuitively obvious." }], "Text"], Cell[TextData[{ "Using the definition of ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:2"], "), substituting in ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " from ", ButtonBox["equation", ButtonData:>"Eq:7", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:7"], "), and using the definition of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:6", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:6"], ") yields" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ StyleBox["G", FontSlant->"Plain"], "=", RowBox[{ SubscriptBox[ StyleBox["G", FontSlant->"Plain"], "0"], "+", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{"log", "[", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox["Z", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], "]"}]}]}]}]}], TraditionalForm]], "NumberedEquation", SpanMaxSize->Infinity, CellTags->"Eq:8"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["G", FontSlant->"Plain"], "0"], TraditionalForm]]], " is a constant. In order to optimise ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], " we envisage two distinct types of change to ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ": a dimensionality preserving perturbation ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[LongRightArrow]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "+", RowBox[{"\[Epsilon]", " ", RowBox[{ StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}], TraditionalForm]]], ", and a dimensionality increasing change ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[LongRightArrow]", RowBox[{"(", RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], ",", RowBox[{ StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}]}], TraditionalForm]]], ". We shall now present the results for these two cases." }], "Text"], Cell[TextData[{ "For perturbations of the form ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[LongRightArrow]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "+", RowBox[{"\[Epsilon]", " ", RowBox[{ StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}], TraditionalForm]]], " we require the functional derivative ", Cell[BoxData[ FormBox[ FractionBox["\[Delta]G", RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], TraditionalForm]]], " which, in turn, requires the results" }], "Text", CellTags->"Ed:Change9"], Cell[BoxData[{ FormBox[ RowBox[{ FractionBox[ RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["p", FontSlant->"Plain"]}]], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"-", RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], RowBox[{ SubscriptBox[ StyleBox["\[Del]", FontWeight->"Bold"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]], StyleBox[" ", FontWeight->"Bold", FontSlant->"Plain"], RowBox[{"\[Delta]", "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ FractionBox[ RowBox[{"\[Delta]Z", "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], "=", "\[AlignmentMarker]", RowBox[{"-", RowBox[{ SubscriptBox[ StyleBox["\[Del]", FontWeight->"Bold"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]], " ", RowBox[{"\[Delta]", "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:9"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["\[Del]", FontWeight->"Bold"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]], TraditionalForm]]], " is the derivative operator wrt ", Cell[BoxData[ FormBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ": these results permit functional differentiation to be replaced by \ ordinary differentiation. After some manipulation we then obtain the \ functional derivative in the form [", ButtonBox["Luttrell, 1988a", ButtonData:>"Ref:Luttrell1988a", ButtonStyle->"Hyperlink"], "]" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ FractionBox["\[Delta]G", RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], FontSlant->"Plain"], "=", RowBox[{ RowBox[{"[", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "-", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "]"}], RowBox[{ SubscriptBox[ StyleBox["\[Del]", FontWeight->"Bold"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]], SubscriptBox[ RowBox[{"log", "[", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox["Z", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], "]"}], RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "=", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]]}]}]}], TraditionalForm]], "NumberedEquation", SpanMaxSize->Infinity, CellTags->"Eq:10"], Cell[TextData[{ "which yields a change ", Cell[BoxData[ \(TraditionalForm\`\[CapitalDelta]G\_1\)]], " in ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], " given by" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ StyleBox[\(\[CapitalDelta]G\_1\), FontSlant->"Plain"], "=", RowBox[{"\[Epsilon]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", "[", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "-", RowBox[{ StyleBox["p", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]}], "]"}], RowBox[{ StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"], ".", RowBox[{ SubscriptBox[ StyleBox["\[Del]", FontWeight->"Bold"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]], RowBox[{"log", "[", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ StyleBox["Z", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]], "]"}]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", SpanMaxSize->Infinity, CellTags->"Eq:11"], Cell["where", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "\[Congruent]", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ StyleBox["p", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], "\[Congruent]", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{ StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:12"], Cell[TextData[{ "Alternatively, for changes of the form ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[LongRightArrow]", RowBox[{"(", RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], ",", RowBox[{ StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], ")"}]}], TraditionalForm]]], " we obtain a change ", Cell[BoxData[ FormBox[ StyleBox[\(\[CapitalDelta]G\_2\), FontSlant->"Plain"], TraditionalForm]]], " in ", Cell[BoxData[ FormBox[ StyleBox["G", FontSlant->"Plain"], TraditionalForm]]], " given by [", ButtonBox["Luttrell, 1988a", ButtonData:>"Ref:Luttrell1988a", ButtonStyle->"Hyperlink"], "]" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ StyleBox[\(\[CapitalDelta]G\_2\), FontSlant->"Plain"], "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Plain"], StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{"log", "[", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], RowBox[{ StyleBox["p", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}]], "]"}]}]}]}], TraditionalForm]], "NumberedEquation", SpanMaxSize->Infinity, CellTags->"Eq:13"], Cell[TextData[{ "Both ", Cell[BoxData[ FormBox[ StyleBox[\(\[CapitalDelta]G\_1\), FontSlant->"Plain"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ StyleBox[\(\[CapitalDelta]G\_2\), FontSlant->"Plain"], TraditionalForm]]], " depend on a comparison of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["p", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], ". Any differences between ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["p", FontSlant->"Plain"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ",", StyleBox["t", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], " are caused by the presence of structure in ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " which is not measured by ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " alone." }], "Text"], Cell[TextData[{ "We have used the techniques outlined in this subsection to design MRF \ coherent image texture models [", ButtonBox["Luttrell, 1987b", ButtonData:>"Ref:Luttrell1987b", ButtonStyle->"Hyperlink"], ",", ButtonBox["c", ButtonData:>"Ref:Luttrell1987c", ButtonStyle->"Hyperlink"], ",", ButtonBox["d", ButtonData:>"Ref:Luttrell1987d", ButtonStyle->"Hyperlink"], "], where we assumed that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " describes spatially stationaxy statistics (i.e. ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox[ RowBox[{ StyleBox["L", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], ")"}], "=", RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], TraditionalForm]]], " where ", Cell[BoxData[ FormBox[ StyleBox["L", FontSlant->"Plain"], TraditionalForm]]], " is any image translation operator). It then suffices to consider only ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " for which a ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox[ RowBox[{ StyleBox["L", FontSlant->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], ")"}], "=", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], TraditionalForm]]], ", which severely restricts the set of feasible ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ". The approach which we have presented in this subsection does not involve \ any hidden variables, so it has difficulty dealing with subtle properties of \ ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " which are better described by introducing \"spectator variables\". \ However it does successfully model short range textural properties." }], "Text", CellTags->"Ed:Change6"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". ", "Cluster decomposition model" }], "Section", CellTags->"Sect:3"], Cell["\<\ We now propose a novel scheme for representing PDFs which completely \ eliminates the need for Monte Carlo simulations, whilst retaining the \ flexibility of the adaptive approach. This improvement is obtained at the \ cost of imposing an artificial hierarchical structure on the PDF \ reconstruction.\ \>", "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " MULTILAYER NEURAL NETWORK" }], "Subsection", CellTags->"Sect:3.1"], Cell["For simplicity consider the following situation", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Congruent]", "\[AlignmentMarker]", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[Congruent]", "\[AlignmentMarker]", RowBox[{"(", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], ",", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}]}], ")"}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:14"], Cell[TextData[{ "Now suppose that the estimated values of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], TraditionalForm]]], ", ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}], TraditionalForm]]], " are used as constraints on a maximum entropy reconstruction ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " (with a uniform prior). After some algebra which is similar to that which \ led to ", ButtonBox["equation", ButtonData:>"Eq:7", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:7"], ") we obtain" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "=", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], RowBox[{ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}], " ", "[", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], ",", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}]}], ")"}], RowBox[{ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], ")"}], RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}], ")"}]}]], "]"}]}]}], TraditionalForm]], "NumberedEquation", SpanMaxSize->Infinity, CellTags->"Eq:15"], Cell[TextData[{ "This expression has a natural interpretation. If ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], TraditionalForm]]], " are independent random variables then so also are ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], TraditionalForm]]], ", yielding ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}], "=", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}]}]}], TraditionalForm]]], ", hence ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "=", RowBox[{ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}]}]}], TraditionalForm]]], ", as expected. On the other hand, if ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], TraditionalForm]]], " are mutually dependent then there is an additional correction term" }], "Text"], Cell[TextData[{ "This approach to estimating ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is usually simpler than specifying ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " directly when ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"dim", "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], "<", RowBox[{"dim", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"dim", "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}], "<", RowBox[{"dim", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}]}], TraditionalForm]]], ". This is because the cost of exhaustively specifying a PDF increases \ exponentially with the dimensionality of its underlying space, so specifying \ three low dimensional PDFs ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ")"}], TraditionalForm]]], ", ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"]}], ")"}], TraditionalForm]]], " is usually cheaper than specifying one high dimensional PDF ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], "." }], "Text"], Cell[TextData[{ "The above decomposition of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " immediately generalises to" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "=", RowBox[{ RowBox[{"[", RowBox[{ UnderoverscriptBox["\[Product]", RowBox[{ StyleBox["i", FontSlant->"Plain"], "=", "1"}], StyleBox["n", FontSlant->"Plain"]], RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], ")"}]}], "]"}], " ", "[", FractionBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], ")"}], RowBox[{ UnderoverscriptBox["\[Product]", RowBox[{ StyleBox["i", FontSlant->"Plain"], "=", "1"}], StyleBox["n", FontSlant->"Plain"]], RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], ")"}], ")"}]}]], "]"}]}], TraditionalForm]], "NumberedEquation", SpanMaxSize->Infinity, CellTags->"Eq:16"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Congruent]", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["n", FontSlant->"Plain"]]}], ")"}]}], TraditionalForm]]], ", ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "\[Congruent]", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["n", FontSlant->"Plain"]]}], ")"}]}], TraditionalForm]]], ", ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontWeight->"Plain", FontSlant->"Plain"]], "\[Congruent]", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontWeight->"Plain", FontSlant->"Plain"]], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontWeight->"Plain", FontSlant->"Plain"]], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"dim", "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontWeight->"Plain", FontSlant->"Plain"]], ")"}], "<", RowBox[{"dim", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontWeight->"Plain", FontSlant->"Plain"]], ")"}]}], TraditionalForm]]], " for ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["i", FontSlant->"Plain"], "=", "1"}], ",", "2", ",", "\[CenterEllipsis]", ",", StyleBox["n", FontSlant->"Plain"]}], TraditionalForm]]], ". Now suppose that ", Cell[BoxData[ FormBox[ RowBox[{"dim", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontWeight->"Plain", FontSlant->"Plain"]], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{"dim", "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontWeight->"Plain", FontSlant->"Plain"]], ")"}], TraditionalForm]]], " are small enough that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], ")"}], TraditionalForm]]], " are easy to estimate (", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["i", FontSlant->"Plain"], "=", "1"}], ",", "2", ",", "\[CenterEllipsis]", ",", StyleBox["n", FontSlant->"Plain"]}], TraditionalForm]]], "), then it remains to estimate ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "1"], ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["n", FontSlant->"Plain"]]}], ")"}], TraditionalForm]]], ". The original problem of estimating ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " has been replaced by an analogous (but simpler) problem of estimating ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " where ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"dim", "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "<", RowBox[{"dim", "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], TraditionalForm]]], ". The maximum entropy procedure may be iterated to yield an estimate of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " itself, and so on until the dimensionalities encountered are low enough \ for a direct estimation of the remaining PDFs to be made. This produces a \ hierarchical cluster decomposition of the original ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " because the layers of sampling functions form a tree-structure. This is a \ type of multilayer neural network." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " TOPOGRAPHIC SAMPLING FUNCTIONS" }], "Subsection", CellTags->"Sect:3.2"], Cell[TextData[{ "The main problem with this type of cluster decomposition is the selection \ of sampling functions. The ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], ")"}], TraditionalForm]]], " must not only be good sampling functions insofar as the statistical \ properties of ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " are concerned, but also the ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["i", FontSlant->"Plain"]], ")"}], TraditionalForm]]], " must stand as reduced dimension representations of the ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ", themselves so that sampling process can be iterated." }], "Text"], Cell[TextData[{ "A novel means of deriving a reduced dimension representation ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " of an input ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " is to use topographic sampling functions [", ButtonBox["Kohonen, 1984", ButtonData:>"Ref:Kohonen1984", ButtonStyle->"Hyperlink"], "]. Define a vector quantisation ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " thus" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "0"], " ", "minimises", " ", SuperscriptBox[ RowBox[{"\[LeftBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "\[RightBracketingBar]"}], "2"], " ", "wrt", " ", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"]}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "0"], "\[Congruent]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], TraditionalForm]}], "NumberedEquation", CellTags->"Eq:17"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is a code book of quantisation vectors parameterised by ", Cell[BoxData[ FormBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ". An update scheme which improves ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " in response to samples ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " drawn from ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[LongRightArrow]", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "+", RowBox[{ RowBox[{"\[Epsilon]", "(", RowBox[{"\[LeftBracketingBar]", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "\[RightBracketingBar]"}], ")"}], " ", "[", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], "]"}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:18"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{"\[Epsilon]", "(", StyleBox["r", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is a non-negative monotonically decreasing function of ", Cell[BoxData[ FormBox[ StyleBox["r", FontSlant->"Plain"], TraditionalForm]]], ". The update function ", Cell[BoxData[ FormBox[ RowBox[{"\[Epsilon]", "(", StyleBox["r", FontSlant->"Plain"], ")"}], TraditionalForm]]], " must have a finite width (in ", Cell[BoxData[ FormBox[ StyleBox["r", FontSlant->"Plain"], TraditionalForm]]], ") to ensure that ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is a continuous function of ", Cell[BoxData[ FormBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ". The converse (", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " a continuous function of ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ") is usually not possible when ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"dim", "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "<", RowBox[{"dim", "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}], TraditionalForm]]], ". Physically ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " can be thought of as a ", Cell[BoxData[ FormBox[ RowBox[{"dim", "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " dimensional manifold embedded in ", Cell[BoxData[ FormBox[ RowBox[{"dim", "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " dimensions. ", ButtonBox["Equation", ButtonData:>"Eq:18", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:18"], ") describes the dynamical behaviour of this manifold in response to being \ \"pulled\" by ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ": the response of the manifold is rather like that of a stiff sheet. A \ particularly desirable property of this learning algorithm is that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["p", FontSlant->"Plain"], "0"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " tends to become constant, thus maximising the output entropy. \ Furthermore, the algorithm can be shown to minimise ", Cell[BoxData[ FormBox[ RowBox[{"<", RowBox[{"log", " ", RowBox[{"(", RowBox[{ StyleBox["V", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], ")"}]}], " ", ">"}], TraditionalForm]]], " where ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["V", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is the error volume associated with the reconstruction of ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " from ", Cell[BoxData[ FormBox[ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " using ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " [", ButtonBox["Luttrell, 1988b", ButtonData:>"Ref:Luttrell1988b", ButtonStyle->"Hyperlink"], "]. It is these properties of the learning algorithm which lead to a ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["s", FontWeight->"Bold", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " being called a topographic sampling function." }], "Text", CellTags->"Ed:Error1"], Cell[TextData[{ "A hierarchy of topographic sampling functions can be derived by extending \ this optimisation scheme. This produces a cluster decomposed representation \ ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["P", FontSlant->"Plain"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["P", FontSlant->"Plain"], "0"], "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " as explained after ", ButtonBox["equation", ButtonData:>"Eq:16", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:16"], "). Furthermore, various improvements can also be introduced which \ enormously speed up the convergence of the update scheme as originally \ proposed [", ButtonBox["Luttrell, 1988c", ButtonData:>"Ref:Luttrell1988c", ButtonStyle->"Hyperlink"], "]." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". Conclusions" }], "Section", CellTags->"Sect:4"], Cell["\<\ There is a pressing need for representations of PDFs in situations where \ direct physical insight fails to provide a complete model. We have discussed \ two alternative techniques: MRFs and cluster decomposition. The MRF technique \ potentially can represent a PDF very accurately by using a sufficiently \ complicated set of potentials and/or hidden vaxiables, but the computational \ cost of Monte Carlo simulation of MRFs can be unacceptable. The cluster \ decomposition technique imposes an artificial hierarchical structure on the \ PDF which can lead to inaccuracies in representation, but it involves no \ Monte Carlo simulations. For real time applications we recommend the use of \ cluster decomposition.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". References" }], "Section"], Cell[TextData[{ "Ackley D H, Hinton G E and Sejnowski T J, 1985, ", StyleBox["Cogn. Sci.", FontSlant->"Italic"], ", ", StyleBox["9", FontWeight->"Bold"], ", 147-169, 'A learning algorithm for Boltzmann machines'." }], "Reference", CellTags->{"Ref:AckleyHintonSejnowski1985", "Ed:Change1"}], Cell[TextData[{ "Besag J, 1974, ", StyleBox["J. R. Statist. Soc", FontSlant->"Italic"], ", ", StyleBox["Ser. B", FontSlant->"Italic"], ", ", StyleBox["36", FontWeight->"Bold"], ", 192-236, 'Spatial interaction and the statistical analysis of lattice \ systems'." }], "Reference", CellTags->"Ref:Besag1974"], Cell[TextData[{ "DeGroot, 1970, ", StyleBox["Optimal statistical decisions", FontSlant->"Italic"], ", New York, McGraw-Hill." }], "Reference", CellTags->"Ref:DeGroot1970"], Cell[TextData[{ "Geman S and Geman D, 1984, ", StyleBox["IEEE PAMI", FontSlant->"Italic"], ", ", StyleBox["6", FontWeight->"Bold"], "(6), 721-741, 'Stochastic relaxation, Gibbs distributions, and the \ Bayesian restoration of images'." }], "Reference", CellTags->"Ref:GemanGeman1984"], Cell[TextData[{ "Geman S and Graffigne C, ", StyleBox["Proc. Int. Cong. Math. 1986", FontSlant->"Italic"], ", Ed. Gleason A M, Am. Math. Soc., Providence, 1987, 'Markov random field \ image models and their applications to computer vision'." }], "Reference", CellTags->"Ref:GemanGraffigne1987"], Cell[TextData[{ "Jaynes E T, 1957, ", StyleBox["Phys. Rev.", FontSlant->"Italic"], ", ", StyleBox["106", FontWeight->"Bold"], "(4), 620-630, 'Information theory and statistical mechanics'." }], "Reference", CellTags->"Ref:Jaynes1957a"], Cell[TextData[{ "-, 1957, ", StyleBox["Phys. Rev.", FontSlant->"Italic"], ", ", StyleBox["108", FontWeight->"Bold"], "(2), 171-190, 'Information theory and statistical mechanics. II'." }], "Reference", CellTags->"Ref:Jaynes1957b"], Cell[TextData[{ "-, 1968, ", StyleBox["IEEE Trans. SSC", FontSlant->"Italic"], ", ", StyleBox["4", FontWeight->"Bold"], "(3), 227-241, 'Prior probabilities'." }], "Reference", CellTags->"Ref:Jaynes1968"], Cell[TextData[{ "-, 1982, ", StyleBox["Proc. IEEE", FontSlant->"Italic"], ", ", StyleBox["70", FontWeight->"Bold"], "(9), 939-952, 'On the rationale of maximum-entropy methods'." }], "Reference", CellTags->"Ref:Jaynes1982"], Cell[TextData[{ "Kindermann R and Snell J L, 1980, ", StyleBox["Markov Random Fields and their Applications", FontSlant->"Italic"], ", ", StyleBox["Contemporary Mathematics", FontSlant->"Italic"], ", Vol. 1, Am. Math. Soc., Providence, Rhode Island." }], "Reference", CellTags->"Ref:KindermannSnell1980"], Cell[TextData[{ "Kohonen T, 1984, ", StyleBox["Self organisation and associative memory", FontSlant->"Italic"], ", Springer-Verlag." }], "Reference", CellTags->"Ref:Kohonen1984"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/3815/3815.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell S P, 1985, ", StyleBox["RSRE Memo.", FontSlant->"Italic"], ", ", StyleBox["3815", FontWeight->"Bold"], ", 'The implications of Boltzmann-type machines for SAR data processing'." }], "Reference", CellTags->"Ref:Luttrell1985"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/mrf/mrf.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "-, 1987a, ", StyleBox["Inv. Prob.", FontSlant->"Italic"], ", ", StyleBox["3", FontWeight->"Bold"], ", 289-300, 'The use of Markov random field models to derive sampling \ schemes for inverse texture problems'." }], "Reference", CellTags->"Ref:Luttrell1987a"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/agard87/agard87.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "-, 1987b, ", StyleBox["Proc. AGARD Conf. on Scattering and Propagation in Random Media", FontSlant->"Italic"], ", 'Markov random fields: a strategy for clutter modelling'." }], "Reference", CellTags->"Ref:Luttrell1987b"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/spie87/spie87.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "-, 1987c, ", StyleBox["Proc. SPIE Int. Symp. on Inverse Problems in Optics", FontSlant->"Italic"], ", ", StyleBox["808", FontWeight->"Bold"], ", 182-188, Ed. Pike E R, 'The use of Markov random field models in \ sampling scheme design'." }], "Reference", CellTags->{"Ed:Change2", "Ref:Luttrell1987c"}], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/radar87/radar87.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "-, 1987d, ", StyleBox["Proc. IEE RADAR-87 Conf.", FontSlant->"Italic"], ", 222-226, 'Designing Markov random field structures for clutter \ modelling'." }], "Reference", CellTags->"Ref:Luttrell1987d"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/sample/sample.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "-, 1988a, to be published in ", StyleBox["Inv. Prob.", FontSlant->"Italic"], ", 'A maximum entropy approach to sampling function design'." }], "Reference", CellTags->"Ref:Luttrell1988a"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/igarss88/igarss88.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "-, 1988b, ", StyleBox["Proc. IGARSS'88 Conf. on Remote Sending Moving Towards the 21st \ Century", FontSlant->"Italic"], ", 'Image compression using a neural network." }], "Reference", CellTags->"Ref:Luttrell1988b"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/compress/compress.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "-, 1988c, submitted to ", StyleBox["Patt. Recog. Letts.", FontSlant->"Italic"], ", 'Image compression using a multilayer neural network'." }], "Reference", CellTags->"Ref:Luttrell1988c"], Cell[TextData[{ "Metropolis N, Rosenbluth A W, Rosenbluth M N, Teller A H and Teller E, \ 1953, ", StyleBox["J. Chem. Phys.", FontSlant->"Italic"], ", ", StyleBox["21", FontWeight->"Bold"], ", 1087-1092, 'Equation of state calculations by fast computing machines'." }], "Reference", CellTags->{ "Ed:Change3", "Ref:MetropolisRosenbluthRosenbluthTellerTeller1953"}], Cell[TextData[{ "Preston C J, 1974, ", StyleBox["Gibbs States on Countable Sets", FontSlant->"Italic"], ", Cambridge University Press." }], "Reference", CellTags->"Ref:Preston1974"], Cell[TextData[{ "Sejnowski T J, 1986, ", StyleBox["Proc. Conf. on Neural networks for Computing", FontSlant->"Italic"], ", Vol. 151, Am. Inst. Phys., Snowbird, Utah, 'Higher order Boltzmann \ machines'." }], "Reference", CellTags->"Ref:Sejnowski1986"] }, Closed]] }, Open ]] }, FrontEndVersion->"5.0 for Microsoft Windows", ScreenRectangle->{{0, 1280}, {0, 941}}, WindowToolbars->{}, WindowSize->{665.375, 641}, WindowMargins->{{307.25, Automatic}, {Automatic, 50}}, Magnification->1, StyleDefinitions -> "Report.nb" ] (******************************************************************* Cached data follows. If you edit this Notebook file directly, not using Mathematica, you must remove the line containing CacheID at the top of the file. The cache data will then be recreated when you save this file from within Mathematica. *******************************************************************) (*CellTagsOutline CellTagsIndex->{ "Eq:1"->{ Cell[15328, 527, 869, 27, 70, "NumberedEquation", CellTags->"Eq:1"]}, "Ed:Change7"->{ Cell[18028, 621, 3426, 114, 70, "Text", CellTags->"Ed:Change7"]}, "Sect:2"->{ Cell[22935, 790, 217, 10, 70, "Section", CellTags->"Sect:2"]}, "Eq:2"->{ Cell[25898, 899, 1349, 38, 70, "NumberedEquation", CellTags->"Eq:2"]}, "Ed:Change4"->{ Cell[27250, 939, 2484, 89, 70, "Text", CellTags->"Ed:Change4"]}, "Sect:2.1"->{ Cell[29759, 1032, 150, 8, 70, "Subsection", CellTags->"Sect:2.1"]}, "Eq:3"->{ Cell[30532, 1067, 3520, 98, 70, "NumberedEquation", CellTags->"Eq:3"]}, "Ed:Change8"->{ Cell[34055, 1167, 2290, 78, 70, "Text", CellTags->"Ed:Change8"]}, "Eq:4"->{ Cell[38321, 1318, 1545, 44, 70, "NumberedEquation", CellTags->{"Eq:4", "Ed:Change5"}]}, "Ed:Change5"->{ Cell[38321, 1318, 1545, 44, 70, "NumberedEquation", CellTags->{"Eq:4", "Ed:Change5"}]}, "Eq:5"->{ Cell[40651, 1396, 2660, 75, 70, "NumberedEquation", CellTags->"Eq:5"]}, "Sect:2.2"->{ Cell[48635, 1651, 149, 8, 70, "Subsection", CellTags->"Sect:2.2"]}, "Eq:6"->{ Cell[50491, 1722, 1308, 39, 70, "NumberedEquation", CellTags->"Eq:6"]}, "Eq:7"->{ Cell[55282, 1882, 2081, 64, 70, "NumberedEquation", CellTags->"Eq:7"]}, "Eq:8"->{ Cell[59644, 2038, 1455, 42, 70, "NumberedEquation", CellTags->"Eq:8"]}, "Ed:Change9"->{ Cell[63634, 2161, 1422, 43, 70, "Text", CellTags->"Ed:Change9"]}, "Eq:9"->{ Cell[65059, 2206, 3077, 92, 70, "NumberedEquation", CellTags->"Eq:9"]}, "Eq:10"->{ Cell[68866, 2325, 2323, 67, 70, "NumberedEquation", CellTags->"Eq:10"]}, "Eq:11"->{ Cell[71444, 2406, 2781, 73, 70, "NumberedEquation", CellTags->"Eq:11"]}, "Eq:12"->{ Cell[74252, 2483, 3748, 110, 70, "NumberedEquation", CellTags->"Eq:12"]}, "Eq:13"->{ Cell[79338, 2640, 1993, 57, 70, "NumberedEquation", CellTags->"Eq:13"]}, "Ed:Change6"->{ Cell[83823, 2787, 3445, 108, 70, "Text", CellTags->"Ed:Change6"]}, "Sect:3"->{ Cell[87317, 2901, 120, 6, 70, "Section", CellTags->"Sect:3"]}, "Sect:3.1"->{ Cell[87790, 2919, 154, 8, 70, "Subsection", CellTags->"Sect:3.1"]}, "Eq:14"->{ Cell[88013, 2931, 1592, 49, 70, "NumberedEquation", CellTags->"Eq:14"]}, "Eq:15"->{ Cell[91566, 3051, 2931, 80, 70, "NumberedEquation", CellTags->"Eq:15"]}, "Eq:16"->{ Cell[101213, 3356, 2307, 66, 70, "NumberedEquation", CellTags->"Eq:16"]}, "Sect:3.2"->{ Cell[111048, 3664, 159, 8, 70, "Subsection", CellTags->"Sect:3.2"]}, "Eq:17"->{ Cell[113858, 3764, 1242, 38, 70, "NumberedEquation", CellTags->"Eq:17"]}, "Eq:18"->{ Cell[116351, 3850, 1526, 45, 70, "NumberedEquation", CellTags->"Eq:18"]}, "Ed:Error1"->{ Cell[117880, 3897, 5310, 177, 70, "Text", CellTags->"Ed:Error1"]}, "Sect:4"->{ Cell[124350, 4118, 98, 5, 70, "Section", CellTags->"Sect:4"]}, "Ref:AckleyHintonSejnowski1985"->{ Cell[125304, 4147, 309, 9, 70, "Reference", CellTags->{"Ref:AckleyHintonSejnowski1985", "Ed:Change1"}]}, "Ed:Change1"->{ Cell[125304, 4147, 309, 9, 70, "Reference", CellTags->{"Ref:AckleyHintonSejnowski1985", "Ed:Change1"}]}, "Ref:Besag1974"->{ Cell[125616, 4158, 334, 13, 70, "Reference", CellTags->"Ref:Besag1974"]}, "Ref:DeGroot1970"->{ Cell[125953, 4173, 183, 6, 70, "Reference", CellTags->"Ref:DeGroot1970"]}, "Ref:GemanGeman1984"->{ Cell[126139, 4181, 304, 10, 70, "Reference", CellTags->"Ref:GemanGeman1984"]}, "Ref:GemanGraffigne1987"->{ Cell[126446, 4193, 306, 7, 70, "Reference", CellTags->"Ref:GemanGraffigne1987"]}, "Ref:Jaynes1957a"->{ Cell[126755, 4202, 255, 9, 70, "Reference", CellTags->"Ref:Jaynes1957a"]}, "Ref:Jaynes1957b"->{ Cell[127013, 4213, 250, 9, 70, "Reference", CellTags->"Ref:Jaynes1957b"]}, "Ref:Jaynes1968"->{ Cell[127266, 4224, 223, 9, 70, "Reference", CellTags->"Ref:Jaynes1968"]}, "Ref:Jaynes1982"->{ Cell[127492, 4235, 243, 9, 70, "Reference", CellTags->"Ref:Jaynes1982"]}, "Ref:KindermannSnell1980"->{ Cell[127738, 4246, 324, 9, 70, "Reference", CellTags->"Ref:KindermannSnell1980"]}, "Ref:Kohonen1984"->{ Cell[128065, 4257, 190, 6, 70, "Reference", CellTags->"Ref:Kohonen1984"]}, "Ref:Luttrell1985"->{ Cell[128258, 4265, 478, 18, 70, "Reference", CellTags->"Ref:Luttrell1985"]}, "Ref:Luttrell1987a"->{ Cell[128739, 4285, 501, 19, 70, "Reference", CellTags->"Ref:Luttrell1987a"]}, "Ref:Luttrell1987b"->{ Cell[129243, 4306, 463, 15, 70, "Reference", CellTags->"Ref:Luttrell1987b"]}, "Ed:Change2"->{ Cell[129709, 4323, 550, 19, 70, "Reference", CellTags->{"Ed:Change2", "Ref:Luttrell1987c"}]}, "Ref:Luttrell1987c"->{ Cell[129709, 4323, 550, 19, 70, "Reference", CellTags->{"Ed:Change2", "Ref:Luttrell1987c"}]}, "Ref:Luttrell1987d"->{ Cell[130262, 4344, 443, 16, 70, "Reference", CellTags->"Ref:Luttrell1987d"]}, "Ref:Luttrell1988a"->{ Cell[130708, 4362, 427, 15, 70, "Reference", CellTags->"Ref:Luttrell1988a"]}, "Ref:Luttrell1988b"->{ Cell[131138, 4379, 467, 17, 70, "Reference", CellTags->"Ref:Luttrell1988b"]}, "Ref:Luttrell1988c"->{ Cell[131608, 4398, 438, 16, 70, "Reference", CellTags->"Ref:Luttrell1988c"]}, "Ed:Change3"->{ Cell[132049, 4416, 385, 11, 70, "Reference", CellTags->{ "Ed:Change3", "Ref:MetropolisRosenbluthRosenbluthTellerTeller1953"}]}, "Ref:MetropolisRosenbluthRosenbluthTellerTeller1953"->{ Cell[132049, 4416, 385, 11, 70, "Reference", CellTags->{ "Ed:Change3", "Ref:MetropolisRosenbluthRosenbluthTellerTeller1953"}]}, "Ref:Preston1974"->{ Cell[132437, 4429, 193, 6, 70, "Reference", CellTags->"Ref:Preston1974"]}, "Ref:Sejnowski1986"->{ Cell[132633, 4437, 263, 7, 70, "Reference", CellTags->"Ref:Sejnowski1986"]} } *) (*CellTagsIndex CellTagsIndex->{ {"Eq:1", 133593, 4466}, {"Ed:Change7", 133691, 4469}, {"Sect:2", 133781, 4472}, {"Eq:2", 133866, 4475}, {"Ed:Change4", 133965, 4478}, {"Sect:2.1", 134056, 4481}, {"Eq:3", 134146, 4484}, {"Ed:Change8", 134246, 4487}, {"Eq:4", 134334, 4490}, {"Ed:Change5", 134450, 4493}, {"Eq:5", 134560, 4496}, {"Sect:2.2", 134658, 4499}, {"Eq:6", 134748, 4502}, {"Eq:7", 134842, 4505}, {"Eq:8", 134936, 4508}, {"Ed:Change9", 135036, 4511}, {"Eq:9", 135124, 4514}, {"Eq:10", 135219, 4517}, {"Eq:11", 135315, 4520}, {"Eq:12", 135411, 4523}, {"Eq:13", 135508, 4526}, {"Ed:Change6", 135609, 4529}, {"Sect:3", 135700, 4532}, {"Sect:3.1", 135789, 4535}, {"Eq:14", 135880, 4538}, {"Eq:15", 135976, 4541}, {"Eq:16", 136072, 4544}, {"Sect:3.2", 136172, 4547}, {"Eq:17", 136264, 4550}, {"Eq:18", 136361, 4553}, {"Ed:Error1", 136462, 4556}, {"Sect:4", 136553, 4559}, {"Ref:AckleyHintonSejnowski1985", 136663, 4562}, {"Ed:Change1", 136796, 4565}, {"Ref:Besag1974", 136932, 4568}, {"Ref:DeGroot1970", 137039, 4571}, {"Ref:GemanGeman1984", 137150, 4574}, {"Ref:GemanGraffigne1987", 137269, 4577}, {"Ref:Jaynes1957a", 137384, 4580}, {"Ref:Jaynes1957b", 137492, 4583}, {"Ref:Jaynes1968", 137599, 4586}, {"Ref:Jaynes1982", 137705, 4589}, {"Ref:KindermannSnell1980", 137820, 4592}, {"Ref:Kohonen1984", 137936, 4595}, {"Ref:Luttrell1985", 138045, 4598}, {"Ref:Luttrell1987a", 138157, 4601}, {"Ref:Luttrell1987b", 138270, 4604}, {"Ed:Change2", 138376, 4607}, {"Ref:Luttrell1987c", 138505, 4610}, {"Ref:Luttrell1987d", 138634, 4613}, {"Ref:Luttrell1988a", 138747, 4616}, {"Ref:Luttrell1988b", 138860, 4619}, {"Ref:Luttrell1988c", 138973, 4622}, {"Ed:Change3", 139079, 4625}, {"Ref:MetropolisRosenbluthRosenbluthTellerTeller1953", 139281, 4629}, {"Ref:Preston1974", 139448, 4633}, {"Ref:Sejnowski1986", 139558, 4636} } *) (*NotebookFileOutline Notebook[{ Cell[CellGroupData[{ Cell[1776, 53, 26, 0, 70, "Section 1"], Cell[CellGroupData[{ Cell[1827, 57, 39, 0, 70, "Subsection"], Cell[1869, 59, 256, 11, 70, "Text"], Cell[2128, 72, 107, 3, 70, "Text"], Cell[2238, 77, 485, 18, 70, "Text"], Cell[2726, 97, 287, 11, 70, "Text"], Cell[3016, 110, 232, 8, 70, "Text"], Cell[3251, 120, 350, 9, 70, "Text"], Cell[3604, 131, 299, 9, 70, "Text"], Cell[3906, 142, 1475, 48, 70, "Text"], Cell[5384, 192, 226, 8, 70, "Text"], Cell[5613, 202, 2339, 74, 70, "Text"], Cell[7955, 278, 1017, 36, 70, "Text"], Cell[8975, 316, 994, 34, 70, "Text"], Cell[9972, 352, 2015, 67, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[12036, 425, 90, 2, 70, "Title"], Cell[12129, 429, 124, 8, 70, "Author"], Cell[12256, 439, 122, 3, 70, "Text"], Cell[12381, 444, 59, 0, 70, "Text"], Cell[12443, 446, 900, 13, 70, "Abstract"], Cell[CellGroupData[{ Cell[13368, 463, 79, 4, 70, "Section 1"], Cell[13450, 469, 1875, 56, 70, "Text"], Cell[15328, 527, 869, 27, 70, "NumberedEquation", CellTags->"Eq:1"], Cell[16200, 556, 1825, 63, 70, "Text"], Cell[18028, 621, 3426, 114, 70, "Text", CellTags->"Ed:Change7"], Cell[21457, 737, 1441, 48, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[22935, 790, 217, 10, 70, "Section", CellTags->"Sect:2"], Cell[23155, 802, 2740, 95, 70, "Text"], Cell[25898, 899, 1349, 38, 70, "NumberedEquation", CellTags->"Eq:2"], Cell[27250, 939, 2484, 89, 70, "Text", CellTags->"Ed:Change4"], Cell[CellGroupData[{ Cell[29759, 1032, 150, 8, 70, "Subsection", CellTags->"Sect:2.1"], Cell[29912, 1042, 617, 23, 70, "Text"], Cell[30532, 1067, 3520, 98, 70, "NumberedEquation", CellTags->"Eq:3"], Cell[34055, 1167, 2290, 78, 70, "Text", CellTags->"Ed:Change8"], Cell[36348, 1247, 1970, 69, 70, "Text"], Cell[38321, 1318, 1545, 44, 70, "NumberedEquation", CellTags->{"Eq:4", "Ed:Change5"}], Cell[39869, 1364, 779, 30, 70, "Text"], Cell[40651, 1396, 2660, 75, 70, "NumberedEquation", CellTags->"Eq:5"], Cell[43314, 1473, 3417, 113, 70, "Text"], Cell[46734, 1588, 1864, 58, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[48635, 1651, 149, 8, 70, "Subsection", CellTags->"Sect:2.2"], Cell[48787, 1661, 1701, 59, 70, "Text"], Cell[50491, 1722, 1308, 39, 70, "NumberedEquation", CellTags->"Eq:6"], Cell[51802, 1763, 3477, 117, 70, "Text"], Cell[55282, 1882, 2081, 64, 70, "NumberedEquation", CellTags->"Eq:7"], Cell[57366, 1948, 1120, 39, 70, "Text"], Cell[58489, 1989, 1152, 47, 70, "Text"], Cell[59644, 2038, 1455, 42, 70, "NumberedEquation", CellTags->"Eq:8"], Cell[61102, 2082, 2529, 77, 70, "Text"], Cell[63634, 2161, 1422, 43, 70, "Text", CellTags->"Ed:Change9"], Cell[65059, 2206, 3077, 92, 70, "NumberedEquation", CellTags->"Eq:9"], Cell[68139, 2300, 724, 23, 70, "Text"], Cell[68866, 2325, 2323, 67, 70, "NumberedEquation", CellTags->"Eq:10"], Cell[71192, 2394, 249, 10, 70, "Text"], Cell[71444, 2406, 2781, 73, 70, "NumberedEquation", CellTags->"Eq:11"], Cell[74228, 2481, 21, 0, 70, "Text"], Cell[74252, 2483, 3748, 110, 70, "NumberedEquation", CellTags->"Eq:12"], Cell[78003, 2595, 1332, 43, 70, "Text"], Cell[79338, 2640, 1993, 57, 70, "NumberedEquation", CellTags->"Eq:13"], Cell[81334, 2699, 2486, 86, 70, "Text"], Cell[83823, 2787, 3445, 108, 70, "Text", CellTags->"Ed:Change6"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[87317, 2901, 120, 6, 70, "Section", CellTags->"Sect:3"], Cell[87440, 2909, 325, 6, 70, "Text"], Cell[CellGroupData[{ Cell[87790, 2919, 154, 8, 70, "Subsection", CellTags->"Sect:3.1"], Cell[87947, 2929, 63, 0, 70, "Text"], Cell[88013, 2931, 1592, 49, 70, "NumberedEquation", CellTags->"Eq:14"], Cell[89608, 2982, 1955, 67, 70, "Text"], Cell[91566, 3051, 2931, 80, 70, "NumberedEquation", CellTags->"Eq:15"], Cell[94500, 3133, 3262, 105, 70, "Text"], Cell[97765, 3240, 3089, 100, 70, "Text"], Cell[100857, 3342, 353, 12, 70, "Text"], Cell[101213, 3356, 2307, 66, 70, "NumberedEquation", CellTags->"Eq:16"], Cell[103523, 3424, 7488, 235, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[111048, 3664, 159, 8, 70, "Subsection", CellTags->"Sect:3.2"], Cell[111210, 3674, 1541, 48, 70, "Text"], Cell[112754, 3724, 1101, 38, 70, "Text"], Cell[113858, 3764, 1242, 38, 70, "NumberedEquation", CellTags->"Eq:17"], Cell[115103, 3804, 1245, 44, 70, "Text"], Cell[116351, 3850, 1526, 45, 70, "NumberedEquation", CellTags->"Eq:18"], Cell[117880, 3897, 5310, 177, 70, "Text", CellTags->"Ed:Error1"], Cell[123193, 4076, 1108, 36, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[124350, 4118, 98, 5, 70, "Section", CellTags->"Sect:4"], Cell[124451, 4125, 738, 11, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[125226, 4141, 75, 4, 70, "Section"], Cell[125304, 4147, 309, 9, 70, "Reference", CellTags->{"Ref:AckleyHintonSejnowski1985", "Ed:Change1"}], Cell[125616, 4158, 334, 13, 70, "Reference", CellTags->"Ref:Besag1974"], Cell[125953, 4173, 183, 6, 70, "Reference", CellTags->"Ref:DeGroot1970"], Cell[126139, 4181, 304, 10, 70, "Reference", CellTags->"Ref:GemanGeman1984"], Cell[126446, 4193, 306, 7, 70, "Reference", CellTags->"Ref:GemanGraffigne1987"], Cell[126755, 4202, 255, 9, 70, "Reference", CellTags->"Ref:Jaynes1957a"], Cell[127013, 4213, 250, 9, 70, "Reference", CellTags->"Ref:Jaynes1957b"], Cell[127266, 4224, 223, 9, 70, "Reference", CellTags->"Ref:Jaynes1968"], Cell[127492, 4235, 243, 9, 70, "Reference", CellTags->"Ref:Jaynes1982"], Cell[127738, 4246, 324, 9, 70, "Reference", CellTags->"Ref:KindermannSnell1980"], Cell[128065, 4257, 190, 6, 70, "Reference", CellTags->"Ref:Kohonen1984"], Cell[128258, 4265, 478, 18, 70, "Reference", CellTags->"Ref:Luttrell1985"], Cell[128739, 4285, 501, 19, 70, "Reference", CellTags->"Ref:Luttrell1987a"], Cell[129243, 4306, 463, 15, 70, "Reference", CellTags->"Ref:Luttrell1987b"], Cell[129709, 4323, 550, 19, 70, "Reference", CellTags->{"Ed:Change2", "Ref:Luttrell1987c"}], Cell[130262, 4344, 443, 16, 70, "Reference", CellTags->"Ref:Luttrell1987d"], Cell[130708, 4362, 427, 15, 70, "Reference", CellTags->"Ref:Luttrell1988a"], Cell[131138, 4379, 467, 17, 70, "Reference", CellTags->"Ref:Luttrell1988b"], Cell[131608, 4398, 438, 16, 70, "Reference", CellTags->"Ref:Luttrell1988c"], Cell[132049, 4416, 385, 11, 70, "Reference", CellTags->{ "Ed:Change3", "Ref:MetropolisRosenbluthRosenbluthTellerTeller1953"}], Cell[132437, 4429, 193, 6, 70, "Reference", CellTags->"Ref:Preston1974"], Cell[132633, 4437, 263, 7, 70, "Reference", CellTags->"Ref:Sejnowski1986"] }, Closed]] }, Open ]] } ] *) (******************************************************************* End of Mathematica Notebook file. *******************************************************************)