(************** Content-type: application/mathematica ************** CreatedBy='Mathematica 5.0' Mathematica-Compatible Notebook This notebook can be used with any Mathematica-compatible application, such as Mathematica, MathReader or Publicon. The data for the notebook starts with the line containing stars above. To get the notebook into a Mathematica-compatible application, do one of the following: * Save the data starting with the line of stars above into a file with a name ending in .nb, then open the file inside the application; * Copy the data starting with the line of stars above to the clipboard, then use the Paste menu command inside the application. Data for notebooks contains only printable 7-bit ASCII and can be sent directly in email or through ftp in text mode. Newlines can be CR, LF or CRLF (Unix, Macintosh or MS-DOS style). NOTE: If you modify the data for this notebook not in a Mathematica- compatible application, you must delete the line below containing the word CacheID, otherwise Mathematica-compatible applications may try to use invalid cache data. For more information on notebooks and Mathematica-compatible applications, contact Wolfram Research: web: http://www.wolfram.com email: info@wolfram.com phone: +1-217-398-0700 (U.S.) Notebook reader applications are available free of charge from Wolfram Research. *******************************************************************) (*CacheID: 232*) (*NotebookFileLineBreakTest NotebookFileLineBreakTest*) (*NotebookOptionsPosition[ 112540, 3696]*) (*NotebookOutlinePosition[ 120728, 3947]*) (* CellTagsIndexPosition[ 119007, 3886]*) (*WindowFrame->Normal*) Notebook[{ Cell[CellGroupData[{ Cell["Notes", "Section 1"], Cell[CellGroupData[{ Cell["Editorial Changes", "Subsection"], Cell[TextData[{ "\"", Cell[BoxData[ FormBox[ SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Italic"], \((\[SelectionPlaceholder])\)], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((\[SelectionPlaceholder])\)], TraditionalForm]]], "\" throughout the paper." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change1", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change2", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change5", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change8", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change9", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change11", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change14", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "Equation reformatted to make it more readable." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change3", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["C", FontSlant->"Italic"]}]], "/", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["Q", FontSlant->"Italic"]}]], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["C", FontSlant->"Italic"]}]], "2"], RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["Q", FontSlant->"Italic"]}]], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change4", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(p\_i\), "(", RowBox[{\(y\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], ")"}], "/", RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{\(p\_i\), "(", RowBox[{\(y\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], ")"}], RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change7", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`\[Times]\)]], "\" inserted at the start of each of the continuation lines." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change6", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`I[X\_\(p\_i\); X\_\(p\_i + 1\); \[CenterEllipsis]\ ; X\_\(p\_\(i + 1\)\)]\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`I[X\_\(p\_i + 1\); X\_\(p\_i + 2\); \[CenterEllipsis]\ ; X\_\(p\_\(i + 1\)\)]\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change10", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ SubsuperscriptBox["x", StyleBox[ RowBox[{ StyleBox["i", FontSlant->"Italic"], "1"}]], \((l)\)], ",", SubsuperscriptBox["x", StyleBox[ RowBox[{ StyleBox["i", FontSlant->"Italic"], "2"}]], \((l)\)], ",", "\[CenterEllipsis]", ",", SubsuperscriptBox["x", SubscriptBox[ StyleBox["in", FontSlant->"Italic"], \(l\_i\)], \((l)\)]}], ")"}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`\((x\_\(i, 1\)\%\((l)\), x\_\(i, 2\)\%\((l)\), \[CenterEllipsis], x\_\(i, n\_\(l\_i\)\)\%\((l)\))\)\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change12", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "Double subscripts separated by a comma throughout this equation." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change13", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"using Equation (28)\" changed to \"using equation (28)\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change15", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(\[Sum]\+\(l = 0\)\%\(L - 1\)\), SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l + 1)\)]}], "-", SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{\(\[Sum]\+\(l = 0\)\%\(L - 1\)\), RowBox[{"(", RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l + 1)\)], "-", SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)]}], ")"}]}], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change16", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"ie\" changed to \"i.e.\"." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["\<\ Bayesian inference on a tree \ \>", "Title"], Cell["\<\ Stephen P Luttrell Pattern Processing Principles SP4 division, RSRE Malvern, WORCS, WR14 3PS\ \>", "Author"], Cell["\<\ This paper appeared as SP4 Research Note, No. 109, 9 February 1990.\ \>", "Text"], Cell["Copyright \[Copyright] Controller HMSO, London, 1990.", "Text"], Cell[TextData[{ StyleBox["Abstract", FontWeight->"Bold"], "\n\n", "Bayesian inference processes information by manipulating probabilities, \ which, in turn, need to be represented in a form that is amenable to both \ adaptive training and manipulation. The Boltzmann machine (and its \ generalisations) is a flexible, but computationally costly, solution to this \ problem. We propose a computationally cheap replacement in the form of a \ cluster decomposition of the state space whose probability needs to be \ represented." }], "Abstract"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Introduction" }], "Section 1"], Cell[TextData[{ "The purpose of this note is to disseminate some theoretical ideas about \ representing joint probability density functions (PDF). Bayesian inference \ [", ButtonBox["1", ButtonData:>"Ref:Jeffreys1939", ButtonStyle->"Hyperlink"], ", ", ButtonBox["2", ButtonData:>"Ref:Cox1946", ButtonStyle->"Hyperlink"], "] is the driving force for constructing PDF representations, and the most \ complete existing solution to the representation problem is the so-called \ \"Boltzmann machine\" [", ButtonBox["3", ButtonData:>"Ref:AckleyHintonSejnowski1985", ButtonStyle->"Hyperlink"], "] (and our own \"Gibbs machine\" [", ButtonBox["4", ButtonData:>"Ref:Luttrell1985", ButtonStyle->"Hyperlink"], ", ", ButtonBox["5", ButtonData:>"Ref:Luttrell1989a", ButtonStyle->"Hyperlink"], "]). However, the computational cost of implementing a Boltzmann machine \ can be horrendous, which motivated us to derive the approach presented in \ this note." }], "Text"], Cell[TextData[{ "In ", ButtonBox["\[Section]", ButtonData:>"Sect:2", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:2"], " we derive a cluster decomposition of a PDF, and its associated maximum \ entropy representation. This derivation leads to a hierarchical network \ structure as the appropriate processor for generating our PDF \ representations. In ", ButtonBox["\[Section]", ButtonData:>"Sect:3", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:3"], " we derive some expressions involving the mutual information between sets \ of nodes in the above hierarchical network. This forms the groundwork for", " ", ButtonBox["\[Section]", ButtonData:>"Sect:4", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:4"], " ", "in which we use relative entropy to assess the quality of the maximum \ entropy PDF representations that we construct." }], "Text"], Cell[TextData[{ "Further details of the relationship between cluster decomposition and the \ Gibbs machine can be found in our review paper [", ButtonBox["6", ButtonData:>"Ref:Luttrell1989b", ButtonStyle->"Hyperlink"], "]." }], "Text"], Cell["\<\ We developed this theory to accompany a hardware image processing project in \ BS1/BS2. Sadly, this project was terminated for reasons beyond my control \ (and beyond my comprehension!). However, we believe that the cluster \ decomposition method of PDF representation (and the associated use of \ Bayesian inference) is sufficiently important that we feel compelled to \ record it for others to use.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " ", "Cluster decomposition" }], "Section", CellTags->"Sect:2"], Cell[TextData[{ "In this section we shall derive a maximum entropy estimate of a \ multidimensional PDF ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ". We shall frequently partition ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " into a number of pieces thus" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], "=", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"]}], ")"}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:1"], Cell[TextData[{ "We shall introduce constraints on certain marginals of ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ", and demand that the estimated PDF ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " should have the maximum entropy subject to these constraints. This yields \ a ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " whose structure reflects the nature of the stated constraints ", StyleBox["alone", FontSlant->"Italic"], "." }], "Text"], Cell[TextData[{ "In ", ButtonBox["\[Section]", ButtonData:>"Sect:2.1", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:2.1"], ".", CounterBox["Subsection", "Sect:2.1"], " we present a simplified derivation in which we partition a joint PDF into \ only two pieces, and thence derive a maximum entropy estimate of ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], ". In ", ButtonBox["\[Section]", ButtonData:>"Sect:2.2", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:2.2"], ".", CounterBox["Subsection", "Sect:2.2"], " we generalise this result to an ", Cell[BoxData[ FormBox[ StyleBox["N", FontSlant->"Plain"], TraditionalForm]]], " piece partition of ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:1"], ")." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", "Two clusters" }], "Subsection", CellTags->"Sect:2.1"], Cell[TextData[{ "Let ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["N", FontSlant->"Plain"], "=", "2"}], TraditionalForm]]], " whence ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], "=", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}], TraditionalForm]]], ", and denote the 2 dimensional PDF as ", Cell[BoxData[ FormBox[ RowBox[{\(P\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], ". Two marginals are then defined by" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{\(P\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], RowBox[{\(P\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:2"], Cell[TextData[{ "Furthermore, let us produce transformed versions of the ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ",", \(i = 1\), ",", "2"}], TraditionalForm]]], " as follows" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], "\[RightTeeArrow]", RowBox[{\(y\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]}], ",", \(i = 1\), ",", "2"}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:3"], Cell[TextData[{ Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(y\_i\), "=", RowBox[{\(y\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]}], ",", \(i = 1\), ",", "2"}], TraditionalForm]]], " may be regarded as reduced representations of their respective ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], TraditionalForm]]], ". Using equation (3) the joint PDF ", Cell[BoxData[ \(TraditionalForm\`\(p\_12\)(y\_1, y\_2)\)]], " of the ", Cell[BoxData[ \(TraditionalForm\`y\_i, i = 1, 2\)]], " is given by" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{\(\(p\_12\)(y\_1, y\_2)\), "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{"\[Delta]", "(", RowBox[{\(y\_1\), "-", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{\(y\_2\), "-", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], RowBox[{\(P\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:4"], Cell[TextData[{ "From ", ButtonBox["equation", ButtonData:>"Eq:4", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:4"], ") two further marginals are defined by" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{\(\(p\_1\) \((y\_1)\)\), "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "2"], \(\(p\_12\)(y\_1, y\_2)\)}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{\(\(p\_2\) \((y\_2)\)\), "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "1"], \(\(p\_12\)(y\_1, y\_2)\)}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:5"], Cell[TextData[{ "We shall use ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], ",", \(i = 1\), ",", "2"}], TraditionalForm]]], " and ", Cell[BoxData[ \(TraditionalForm\`\(p\_12\)(y\_1, y\_2)\)]], " (and its marginals in ", ButtonBox["equation", ButtonData:>"Eq:5", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:5"], ")) to constrain the maximum entropy estimate below." }], "Text"], Cell["Define the entropy functional", "Text"], Cell[BoxData[ FormBox[ RowBox[{\(H\_2\), "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], RowBox[{"log", "[", RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], "]"}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:6"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], " denotes an estimate of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], ". Now define a cost function to impose the required constraints on the \ marginal PDFs" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{\(C\_2\), "=", RowBox[{\(H\_2\), "+", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{ RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], " ", "[", RowBox[{ RowBox[{\(\[Lambda]\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "+", RowBox[{\(\[Lambda]\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], "]"}]}]}], "+", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "2"], RowBox[{\(\[Mu](y\_1, y\_2)\), " ", "[", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{"\[Delta]", "(", RowBox[{\(y\_1\), "-", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{\(y\_2\), "-", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}]}], "]"}]}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->{"Ed:Change1", "Eq:7"}], Cell[TextData[{ "and where we use Lagrange multiplier functions ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(\[Lambda]\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], ",", \(i = 1\), ",", "2"}], TraditionalForm]]], " and ", Cell[BoxData[ \(TraditionalForm\`\[Mu](y\_1, y\_2)\)]], " to introduce the required constraints on ", Cell[BoxData[ FormBox[ RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], ". Functionally differentiating ", Cell[BoxData[ \(TraditionalForm\`C\_2\)]], " yields" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["C", FontSlant->"Italic"]}]], "2"], RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["Q", FontSlant->"Italic"]}]], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]], "=", RowBox[{\(-1\), "-", RowBox[{"log", "[", RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], "]"}], "+", RowBox[{\(\[Lambda]\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "+", RowBox[{\(\[Lambda]\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "+", RowBox[{"\[Mu]", "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}]}]}], TraditionalForm]], "NumberedEquation", CellTags->{"Ed:Change2", "Eq:8"}], Cell[TextData[{ "Let us denote the maximum entropy solution satisfying ", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["C", FontSlant->"Italic"]}]], "2"], RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["Q", FontSlant->"Italic"]}]], "12"], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]], "=", "0"}], TraditionalForm]]], " as ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["P", RowBox[{ StyleBox["mem", FontSlant->"Italic"], StyleBox[",", FontSlant->"Plain"], StyleBox["12", FontSlant->"Plain"]}]], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], ", which is thus given by" }], "Text", CellTags->"Ed:Change3"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{ SubscriptBox["P", RowBox[{ StyleBox["mem", FontSlant->"Italic"], StyleBox[",", FontSlant->"Plain"], StyleBox["12", FontSlant->"Plain"]}]], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], "\[Proportional]", "\[AlignmentMarker]", RowBox[{"exp", "[", RowBox[{ RowBox[{\(\[Lambda]\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "+", RowBox[{\(\[Lambda]\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "+", RowBox[{"\[Mu]", "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}]}], "]"}]}], "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], FractionBox[ RowBox[{\(p\_12\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], RowBox[{ RowBox[{\(p\_1\), "(", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ")"}], RowBox[{\(p\_2\), "(", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], ")"}]}]]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:9"], Cell[TextData[{ "The final result in ", ButtonBox["equation", ButtonData:>"Eq:9", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:9"], ") is both correctly normalised and satisfies the stated constraints." }], "Text"], Cell[TextData[{ ButtonBox["Equation", ButtonData:>"Eq:9", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:9"], ") has an intuitively obvious form. The ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], TraditionalForm]]], " factor is the solution that we would expect if the constraints on ", Cell[BoxData[ \(TraditionalForm\`\(p\_12\)(y\_1, y\_2)\)]], " were omitted. This is because there would then be no information about \ correlations between ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], ". The remaining factor we shall term the ", StyleBox["joining factor", FontSlant->"Italic"], ", because it introduces knowledge of correlations between ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], ". The normalisation of the joining factor ensures that it contains only \ correlation information. For instance, it is unity when ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], " are independent, and any value other than unity (for any pair ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], " indicates dependence." }], "Text"], Cell[TextData[{ "We may write ", ButtonBox["equation", ButtonData:>"Eq:9", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:9"], ") in the following alternative form" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox["P", RowBox[{ StyleBox["mem", FontSlant->"Italic"], StyleBox[",", FontSlant->"Plain"], StyleBox["12", FontSlant->"Plain"]}]], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], "=", RowBox[{ RowBox[{\(p\_12\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], FractionBox[ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{\(p\_1\), "(", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ")"}]], FractionBox[ RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], RowBox[{\(p\_2\), "(", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], ")"}]]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:10"], Cell[TextData[{ "Here the ", Cell[BoxData[ FormBox[ RowBox[{\(p\_12\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], TraditionalForm]]], " factor approximates the joint PDF of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], " when factors to correct for the number ", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{\(p\_i\), "(", RowBox[{\(y\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], ")"}], RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], TraditionalForm]]], " which map to ", Cell[BoxData[ \(TraditionalForm\`y\_i\)]], " have been divided out (for ", Cell[BoxData[ \(TraditionalForm\`i = 1, 2\)]], ")." }], "Text", CellTags->"Ed:Change4"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", Cell[BoxData[ FormBox[ StyleBox["N", FontSlant->"Plain"], TraditionalForm]]], " clusters" }], "Subsection", CellTags->"Sect:2.2"], Cell[TextData[{ "We shall now extend the results of ", ButtonBox["\[Section]", ButtonData:>"Sect:2.1", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:2.1"], ".", CounterBox["Subsection", "Sect:2.1"], " to ", Cell[BoxData[ FormBox[ StyleBox["N", FontSlant->"Plain"], TraditionalForm]]], " partitions of ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ". No further derivations are needed because the details are almost \ identical to those in ", ButtonBox["\[Section]", ButtonData:>"Sect:2.1", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:2.1"], ".", CounterBox["Subsection", "Sect:2.1"], ". The results which correspond to ", ButtonBox["equation", ButtonData:>"Eq:2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:2"], "), ", ButtonBox["equation", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:3"], "), ", ButtonBox["equation", ButtonData:>"Eq:4", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:4"], ") and ", ButtonBox["equation", ButtonData:>"Eq:5", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:5"], ") are respectively" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], "\[CenterEllipsis]", " ", OverscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "i"], "^"], "\[CenterEllipsis]", " ", SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], \(N - 1\)], RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}]}], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", "N"}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:11"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], "\[RightTeeArrow]", RowBox[{\(y\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]}], ",", \(i = 1\), ",", "N"}], TraditionalForm]], "NumberedEquation"], Cell[BoxData[ FormBox[ RowBox[{\(\(p\_\(12 \[CenterEllipsis]\ N\)\)(y\_1, y\_2, \[CenterEllipsis], y\_N)\), "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], "\[CenterEllipsis]", " ", SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "N"], RowBox[{"\[Delta]", "(", RowBox[{\(y\_1\), "-", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{\(y\_2\), "-", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], RowBox[{"\[CenterEllipsis]\[Delta]", "(", RowBox[{\(y\_N\), "-", RowBox[{\(y\_N\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}]}], ")"}], RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Ed:Change5"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(\(p\_i\)(y\_i)\), "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "2"], "\[CenterEllipsis]", " ", OverscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "i"], "^"], "\[CenterEllipsis]", " ", SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "N"], \(\(p\_\(12 \[CenterEllipsis]\ N\)\)(y\_1, y\_2, \[CenterEllipsis], y\_N)\)}]}]}], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", "N"}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:14"], Cell[TextData[{ "where the hats in ", ButtonBox["equation", ButtonData:>"Eq:11", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:11"], ") and ", ButtonBox["equation", ButtonData:>"Eq:14", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:14"], ") denote an omitted integral." }], "Text"], Cell[TextData[{ "In this case we shall use a generalisation of ", ButtonBox["equation", ButtonData:>"Eq:6", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:6"], "), ", ButtonBox["equation", ButtonData:>"Eq:7", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:7"], ") and ", ButtonBox["equation", ButtonData:>"Eq:8", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:8"], ") to ", Cell[BoxData[ FormBox[ StyleBox["N", FontSlant->"Plain"], TraditionalForm]]], " partitions of ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ", together with ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", "N"}], TraditionalForm]]], " and ", Cell[BoxData[ \(TraditionalForm\`\(p\_\(12 \[CenterEllipsis]\ N\)\)(y\_1, y\_2, \[CenterEllipsis], y\_N)\)]], " (and its marginals in ", ButtonBox["equation", ButtonData:>"Eq:14", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:14"], ")) to constrain the maximum entropy estimate. This leads to the analogue \ of ", ButtonBox["equation", ButtonData:>"Eq:9", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:9"], ") which gives the maximum entropy estimate ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "\[AlignmentMarker]", "=", RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(P\_N\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}], FractionBox[ RowBox[{\(p\_\(12 \[CenterEllipsis]\ N\)\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], ",", "\[CenterEllipsis]", ",", RowBox[{\(y\_N\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}]}], ")"}], RowBox[{ RowBox[{\(p\_1\), "(", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ")"}], RowBox[{\(p\_2\), "(", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(p\_N\), "(", RowBox[{\(y\_N\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}], ")"}]}]]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:15"], Cell[TextData[{ "The interpretation of ", ButtonBox["equation", ButtonData:>"Eq:15", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:15"], ") is analogous that of ", ButtonBox["equation", ButtonData:>"Eq:9", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:9"], ") (and also ", ButtonBox["equation", ButtonData:>"Eq:10", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:10"], "))." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", "Hierarchical clusters" }], "Subsection"], Cell[TextData[{ "We now have all of the theoretical machinery in place for a hierarchical \ decomposition of ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ". The net effect of using ", ButtonBox["equation", ButtonData:>"Eq:15", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:15"], ") to estimate ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is to reduce the dimensionality of the PDFs which have to be estimated. \ Thus ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " is ", Cell[BoxData[ FormBox[ RowBox[{"dim", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], " dimensional (", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"dim", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], "=", RowBox[{\(\[Sum]\_\(i = 1\)\%N\), RowBox[{"dim", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]}]}], TraditionalForm]]], "), whereas ", Cell[BoxData[ FormBox[ RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], TraditionalForm]]], " is ", Cell[BoxData[ FormBox[ RowBox[{"dim", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], TraditionalForm]]], " dimensional (", Cell[BoxData[ \(TraditionalForm\`i = 1, 2, \[CenterEllipsis], N\)]], ") and ", Cell[BoxData[ \(TraditionalForm\`\(p\_\(12 \[CenterEllipsis]\ N\)\)(y\_1, y\_2, \[CenterEllipsis], y\_N)\)]], " is ", Cell[BoxData[ \(TraditionalForm\`N\)]], " dimensional. Each PDF can be represented as a histogram whose number of \ bins depends exponentially on its dimension. The dimension reduction we \ achieve by using a maximum entropy estimate can make the histogram approach \ feasible." }], "Text"], Cell[TextData[{ "A general strategy for partitioning ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " is, to select ", Cell[BoxData[ \(TraditionalForm\`N\)]], " and the ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", "N"}], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:1"], ") to ensure that the ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"dim", "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", "N"}], TraditionalForm]]], " are acceptably small, so that the ", Cell[BoxData[ FormBox[ RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:15", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:15"], ") may be easily represented using histograms. This leaves the ", Cell[BoxData[ \(TraditionalForm\`N\)]], " dimensional ", Cell[BoxData[ \(TraditionalForm\`\(p\_\(12 \[CenterEllipsis]\ N\)\)(y\_1, y\_2, \[CenterEllipsis], y\_N)\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:15", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:15"], ") to represent as a histogram (its marginals ", Cell[BoxData[ \(TraditionalForm\`\(p\_i\)(y\_i), i = 1, 2, \[CenterEllipsis], N\)]], " may also be obtained from this histogram)." }], "Text"], Cell[TextData[{ "If ", Cell[BoxData[ \(TraditionalForm\`N\)]], " is too large for a histogram representation of ", Cell[BoxData[ \(TraditionalForm\`\(p\_\(12 \[CenterEllipsis]\ N\)\)(y\_1, y\_2, \[CenterEllipsis], y\_N)\)]], " to be formed, then we must find an approximation which involves PDFs with \ a smaller dimension. This is precisely the problem that we started with, \ except that we must now estimate ", Cell[BoxData[ \(TraditionalForm\`\(p\_\(12 \[CenterEllipsis]\ N\)\)(y\_1, y\_2, \[CenterEllipsis], y\_N)\)]], " rather than ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ". We use the same maximum entropy procedure to provide us with a suitable \ estimate, and similarly any further estimates should the dimension need to be \ be reduced again, and so on." }], "Text"], Cell["\<\ We shall now present the final maximum entropy expression which applies to an \ arbitrary number of dimension reducing stages.\ \>", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"[", RowBox[{ FractionBox[ RowBox[{\(P\_1\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((0)\)], ")"}], \(\(p\_1\%\((1)\)\)(y\_1\%\((1)\))\)], FractionBox[ RowBox[{\(P\_2\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((0)\)], ")"}], \(\(p\_2\%\((1)\)\)(y\_2\%\((1)\))\)], "\[CenterEllipsis]", FractionBox[ RowBox[{\(P\_\(N\_0\)\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_0\), \((0)\)], ")"}], \(\(p\_\(N\_0\)\%\((1)\)\)( y\_\(N\_0\)\%\((1)\))\)]}], "]"}], "\[IndentingNewLine]", "\[Times]", "\[AlignmentMarker]", RowBox[{"[", RowBox[{ FractionBox[ RowBox[{\(P\_1\%\((1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((1)\)], ")"}], \(\(p\_1\%\((2)\)\)(y\_1\%\((2)\))\)], FractionBox[ RowBox[{\(P\_2\%\((1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((1)\)], ")"}], \(\(p\_2\%\((2)\)\)(y\_2\%\((2)\))\)], "\[CenterEllipsis]", FractionBox[ RowBox[{\(P\_\(N\_1\)\%\((1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_1\), \((1)\)], ")"}], \(\(p\_\(N\_1\)\%\((2)\)\)( y\_\(N\_1\)\%\((2)\))\)]}], "]"}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ "\[VerticalEllipsis]", "\[AlignmentMarker]", "\[IndentingNewLine]", "\[Times]", "\[AlignmentMarker]", RowBox[{"[", RowBox[{ FractionBox[ RowBox[{\(P\_1\%\((L - 1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((L - 1)\)], ")"}], \(\(p\_1\%\((L)\)\)(y\_1\%\((L)\))\)], FractionBox[ RowBox[{\(P\_2\%\((L - 1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((L - 1)\)], ")"}], \(\(p\_2\%\((L)\)\)(y\_2\%\((L)\))\)], "\[CenterEllipsis]", FractionBox[ RowBox[{\(P\_\(N\_\(L - 1\)\)\%\((L - 1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_\(L - 1\)\), \((L - 1)\)], ")"}], \(\(p\_\(N\_\(L - 1\)\)\%\((L)\)\)( y\_\(N\_\(L - 1\)\)\%\((L)\))\)]}], "]"}], "\[IndentingNewLine]", "\[Times]", "\[AlignmentMarker]", \(\(p\_\(12 \[CenterEllipsis]\ N\_\(L - \ 1\)\)\%\((L)\)\) \((y\_1\%\((L)\), y\_2\%\((L)\), \[CenterEllipsis], y\_\(N\_\(L - 1\)\)\%\((L)\))\)\)}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->{"Eq:16", "Ed:Change7"}], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((l)\)], "=", RowBox[{"(", RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((l)\)], ",", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((l)\)], ",", "\[CenterEllipsis]", ",", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_l\), \((l)\)]}], ")"}]}], ",", \(l = 0\), ",", "1", ",", "\[CenterEllipsis]", ",", \(L - 1\)}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:17"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(y\_i\%\((l)\)\), "=", RowBox[{\(y\_i\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l - 1)\)], ")"}]}], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", \(N\_\(L - 1\)\), ",", \(l = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", "L"}], TraditionalForm]], "NumberedEquation"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((l)\)], "=", \((y\_1\%\((l)\), y\_2\%\((l)\), \[CenterEllipsis], y\_\(N\_\(l - 1\)\)\%\((l)\))\)}], ",", \(l = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", \(L - 1\)}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:19"], Cell[TextData[{ "The ", Cell[BoxData[ \(TraditionalForm\`L = 1\)]], " case result was presented in ", ButtonBox["equation", ButtonData:>"Eq:15", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:15"], "), where ", Cell[BoxData[ \(TraditionalForm\`N = N\_0\)]], ", ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], "=", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)]}], TraditionalForm]]], ", ", Cell[BoxData[ FormBox[ RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((0)\)], "=", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"]}], TraditionalForm]]], ", ", Cell[BoxData[ \(TraditionalForm\`y\_i\%\((1)\) = y\_i\)]], ", ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_i\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((0)\)], ")"}], "=", RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ \(TraditionalForm\`\(p\_i\%\((1)\)\)(y\_i\%\((1)\)) = \(p\_i\)( y\_i)\)]], " (for ", Cell[BoxData[ \(TraditionalForm\`i = 1, 2, \[CenterEllipsis], N\)]], ")." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " ", "Mutual information" }], "Section", CellTags->"Sect:3"], Cell[TextData[{ "In ", ButtonBox["\[Section]", ButtonData:>"Sect:2", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:2"], " we saw how the partitioning of state vectors together with a maximum \ entropy procedure led to a convenient representation (", ButtonBox["equation", ButtonData:>"Eq:16", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:16"], ")) of a high dimensional PDF ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], ")"}], TraditionalForm]]], ". In this section we shall develop some theoretical results (in terms of \ mutual information) that can be used to characterise the quality of the \ maximum entropy estimate (see ", ButtonBox["\[Section]", ButtonData:>"Sect:4", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:4"], ")." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", "Basic definitions" }], "Subsection"], Cell[TextData[{ "The basic definition of the mutual information ", Cell[BoxData[ \(TraditionalForm\`I[X; Y]\)]], " between a pair of random variables ", Cell[BoxData[ \(TraditionalForm\`X\)]], " and ", Cell[BoxData[ \(TraditionalForm\`Y\)]], " may be given in three equivalent forms" }], "Text"], Cell[BoxData[ \(TraditionalForm\`I[X; Y] = H[X] + H[Y] - H[X, Y]\)], "NumberedEquation",\ CellTags->"Eq:20"], Cell[BoxData[ \(TraditionalForm\`\(\(\t\)\(\(=\)\(H[X] - H[X | Y]\)\)\)\)], "NumberedEquation", CellTags->"Eq:21"], Cell[BoxData[ \(TraditionalForm\`\(\(\t\)\(\(=\)\(H[Y] - H[Y | X]\)\)\)\)], "NumberedEquation", CellTags->"Eq:22"], Cell[TextData[{ "where ", Cell[BoxData[ \(TraditionalForm\`H[r]\)]], " denotes the entropy functional applied to the PDF of random variable ", Cell[BoxData[ \(TraditionalForm\`r\)]], ". ", Cell[BoxData[ \(TraditionalForm\`e\^H[r]\)]], " is the effective number of states which is available to ", Cell[BoxData[ \(TraditionalForm\`r\)]], ", which we shall define as ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", "r", "]"}], TraditionalForm]]], ". Thus ", Cell[BoxData[ \(TraditionalForm\`e\^\(H[X] + H[Y]\)\)]], " is the value of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X, Y\), "]"}], TraditionalForm]]], " ignoring any mutual dependencies between ", Cell[BoxData[ \(TraditionalForm\`X\)]], " and ", Cell[BoxData[ \(TraditionalForm\`Y\)]], ", and ", Cell[BoxData[ \(TraditionalForm\`e\^H[X, Y]\)]], " is the same including dependencies. Thus ", Cell[BoxData[ \(TraditionalForm\`e\^I[X; Y]\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:20", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:20"], ") measures the reduction in ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X, Y\), "]"}], TraditionalForm]]], " which occurs when mutual dependencies between ", Cell[BoxData[ \(TraditionalForm\`X\)]], " and ", Cell[BoxData[ \(TraditionalForm\`Y\)]], " are accounted for. This property is the basic reason for the term mutual \ information. The two alternative definitions in ", ButtonBox["equation", ButtonData:>"Eq:21", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:21"], ") and ", ButtonBox["equation", ButtonData:>"Eq:22", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:22"], ") may be similarly interpreted, although ", Cell[BoxData[ \(TraditionalForm\`X\)]], " and ", Cell[BoxData[ \(TraditionalForm\`Y\)]], " now enter asymmetrically." }], "Text"], Cell[TextData[{ "The most important property of mutual information is that it is an ", StyleBox["objective", FontSlant->"Italic"], " measure of correlations, as required of a consistent measure of \ information. Therefore it treats correlations, anti-correlations (and \ anything else) equivalently provided that they all lead to the same reduction \ in ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X, Y\), "]"}], TraditionalForm]]], ". Distinctions between various types of correlations made on the basic of \ measures other than ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X, Y\), "]"}], TraditionalForm]]], " are necessarily ", StyleBox["subjective", FontSlant->"Italic"], "." }], "Text"], Cell[TextData[{ "We may generalise", " ", ButtonBox["equation", ButtonData:>"Eq:20", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:20"], ")", " somewhat as follows" }], "Text"], Cell[BoxData[ \(TraditionalForm\`I[X\_1; X\_2; \[CenterEllipsis]; X\_n] = \[Sum]\+\(i = 1\)\%N H[X\_i] - H[X\_1, X\_2, \[CenterEllipsis], X\_n]\)], "NumberedEquation", CellTags->"Eq:23"], Cell[TextData[{ Cell[BoxData[ \(TraditionalForm\`e\^I[X\_1; X\_2; \[CenterEllipsis]; X\_n]\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:23", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:23"], ") measures the reduction in ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X\_1, X\_2, \[CenterEllipsis], X\_n\), "]"}], TraditionalForm]]], " when mutual dependencies amongst the ", Cell[BoxData[ \(TraditionalForm\`X\_i, i = 1, 2, \[CenterEllipsis], N\)]], " are taken into account" }], "Text"], Cell["\<\ We may further generalise these results by partitioning the random variables \ into mutually exclusive sets\ \>", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], "=", "\[AlignmentMarker]", \((X\_1, X\_2, \[CenterEllipsis], X\_n)\)}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "=", "\[AlignmentMarker]\[AlignmentMarker]", \((X\_1, X\_2, \[CenterEllipsis], X\_\(p\_1\))\)}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "=", "\[AlignmentMarker]\[AlignmentMarker]", \((X\_\(p\_1 + 1\), X\_\(p\_1 + 2\), \[CenterEllipsis], X\_\(p\_2\))\)}], TraditionalForm], "\n", FormBox[ RowBox[{"\[VerticalEllipsis]", "\[AlignmentMarker]"}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], "=", "\[AlignmentMarker]\[AlignmentMarker]", \((X\_\(p\_\(N - 1\) + 1\), X\_\(p\_\(N - 1\) + 2\), \[CenterEllipsis], X\_n)\)}], TraditionalForm], "\n", FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], "=", "\[AlignmentMarker]", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"]}], ")"}]}], TraditionalForm], "\n", FormBox[\(p\_0 = \[AlignmentMarker]0\), TraditionalForm], "\n", FormBox[\(p\_N = \[AlignmentMarker]n\), TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:24"], Cell[BoxData[ FormBox[ RowBox[{\(I[X\_1; X\_2; \[CenterEllipsis]; X\_n]\), "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"I", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ";", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ";", "\[CenterEllipsis]", ";", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"]}], "]"}], "+", \(\[Sum]\+\(i = 0\)\%\(N - 1\)I[X\_\(p\_i + 1\); X\_\(p\_i + 2\); \[CenterEllipsis]\ ; X\_\(p\_\(i + 1\)\)]\)}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->{"Ed:Change6", "Eq:25"}], Cell[TextData[{ "Note that ", ButtonBox["equation", ButtonData:>"Eq:24", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:24"], ") makes connection with the notation of ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:1"], "), and that we have defined dummy partition boundaries ", Cell[BoxData[ \(TraditionalForm\`p\_0\)]], " and ", Cell[BoxData[ \(TraditionalForm\`p\_N\)]], " merely for convenience. ", ButtonBox["Equation", ButtonData:>"Eq:25", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:25"], ") is very intuitive, because it states that the total mutual information \ between all components ", Cell[BoxData[ \(TraditionalForm\`X\_i, i = 1, 2, \[CenterEllipsis], n\)]], " of ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " may be divided into two parts. The first part is the mutual information \ between partitions ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", "N"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ". The second part is the mutual information between the components ", Cell[BoxData[ \(TraditionalForm\`\((X\_\(p\_i + 1\); X\_\(p\_i + 2\); \[CenterEllipsis]\ ; X\_\(p\_\(i + 1\)\))\)\)]], " of a partition, summed over partitions ", Cell[BoxData[ \(TraditionalForm\`i = 0, 2, \[CenterEllipsis], N - 1\)]], ". We shall call these the ", StyleBox["inter-partition", FontSlant->"Italic"], " and the ", StyleBox["intra-partition", FontSlant->"Italic"], " mutual information, respectively." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", "Cluster decomposition of mutual information" }], "Subsection"], Cell[TextData[{ "We shall use ", ButtonBox["equation", ButtonData:>"Eq:25", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:25"], ") to analyse the structure of ", ButtonBox["equation", ButtonData:>"Eq:16", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:16"], "). Comparing the notation in ", ButtonBox["equation", ButtonData:>"Eq:24", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:24"], ") with that in ", ButtonBox["equation", ButtonData:>"Eq:19", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:19"], ") leads us to define" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l)\)], "=", "\[AlignmentMarker]", RowBox[{\(I[y\_1\%\((l)\); y\_2\%\((l)\); \[CenterEllipsis]; y\_\(N\_\(l - 1\)\)\%\((l)\)]\), "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "1", \((l)\)], SubsuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], "2", \((l)\)], "\[CenterEllipsis]", " ", SubsuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["y", FontSlant->"Italic"]}]], \(N\_\(l - 1\)\), \((l)\)], \(\(p\_\(12 \[CenterEllipsis]\ \ N\_\(l - 1\)\)\%\((l)\)\)(y\_1\%\((l)\), y\_2\%\((l)\), \[CenterEllipsis], y\_\(N\_\(l - 1\)\)\%\((l)\))\), \(log[\(\(p\_\(12 \ \[CenterEllipsis]\ N\_\(l - 1\)\)\%\((l)\)\)(y\_1\%\((l)\), y\_2\%\((l)\), \ \[CenterEllipsis], y\_\(N\_\(l - \ 1\)\)\%\((l)\))\)\/\(\(\(p\_1\%\((l)\)\)(y\_1\%\((l)\))\) \ \(\(p\_2\%\((l)\)\)(y\_2\%\((l)\))\) \[CenterEllipsis]\ \(\(p\_\(N\_\(l - 1\)\ \)\%\((l)\)\)(y\_\(N\_\(l - 1\)\)\%\((l)\))\)\)]\), " ", "l"}]}], "=", "1"}]}]}], ",", "2", ",", "\[CenterEllipsis]", ",", "L"}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->{"Ed:Change8", "Eq:26"}], Cell[TextData[{ "Similarly, comparing notation with ", ButtonBox["equation", ButtonData:>"Eq:17", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:17"], ") leads us to define" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"I", "[", RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((l)\)], ";", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((l)\)], ";", "\[CenterEllipsis]", ";", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_l\), \((l)\)]}], "]"}], "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1", \((l)\)], SubsuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2", \((l)\)], "\[CenterEllipsis]", " ", SubsuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], \(N\_l\), \((l)\)], RowBox[{\(P\^l\), "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((l)\)], ")"}], RowBox[{"log", "[", FractionBox[ RowBox[{\(P\^l\), "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((l)\)], ")"}], RowBox[{ RowBox[{\(P\_1\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((l)\)], ")"}], RowBox[{\(P\_2\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((l)\)], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(P\_\(N\_l\)\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_l\), \((l)\)], ")"}]}]], "]"}], " ", "l"}]}], "=", "0"}]}]}], ",", "1", ",", "\[CenterEllipsis]", ",", \(L - 1\)}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->{"Ed:Change9", "Eq:27"}], Cell[TextData[{ Cell[BoxData[ FormBox[ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l)\)], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:26", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:26"], ") and ", Cell[BoxData[ FormBox[ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:27", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:27"], ") are the total mutual information between components of layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], ", and the inter-partition mutual information of layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], ", respectively." }], "Text"], Cell["The following inequality is satisfied", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], "\[GreaterEqual]", SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l + 1)\)], "\[GreaterEqual]", "0"}], ",", " ", \(l = 0\), ",", "1", ",", "\[CenterEllipsis]", ",", \(L - 1\)}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:28"], Cell[TextData[{ "which can be proved by applying Jensen's inequality to the convex ", Cell[BoxData[ \(TraditionalForm\`\[Intersection]\)]], " logarithm function. ", ButtonBox["Equation", ButtonData:>"Eq:28", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:28"], ") states that a gain in mutual information between a set of ", Cell[BoxData[ FormBox[ RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l)\)], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", \(N\_l\)}], TraditionalForm]]], " cannot be achieved applying the mappings ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(y\_i\%\((l + 1)\)\), "=", RowBox[{\(y\_i\%\((l + 1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l)\)], ")"}]}], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", \(N\_l\)}], TraditionalForm]]], ", and then measuring the mutual information between the ", Cell[BoxData[ \(TraditionalForm\`y\_i\%\((l + 1)\), i = 1, 2, \[CenterEllipsis], N\_l\)]], "." }], "Text"], Cell[TextData[{ "We can obtain an equation which is analogous to ", ButtonBox["equation", ButtonData:>"Eq:25", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:25"], ") by defining" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{\(N\_\(-1\)\), "=", "\[AlignmentMarker]", RowBox[{"dim", "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}]}], TraditionalForm], "\n", FormBox[ RowBox[{ SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], "=", "\[AlignmentMarker]", \((y\_1\%\((0)\), y\_2\%\((0)\), \[CenterEllipsis], y\_\(N\_\(-1\)\)\%\((0)\))\)}], TraditionalForm], "\n", FormBox[ RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((0)\)], "=", "\[AlignmentMarker]", \(I[y\_1\%\((0)\); y\_2\%\((0)\); \[CenterEllipsis]; y\_\(N\_\(-1\)\)\%\((0)\)]\)}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker], Cell[TextData[{ "which extends ", ButtonBox["equation", ButtonData:>"Eq:19", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:19"], ") and ", ButtonBox["equation", ButtonData:>"Eq:26", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:26"], ") to include the case ", Cell[BoxData[ \(TraditionalForm\`l = 0\)]], ". We also define" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l)\)], "=", \((x\_\(i, 1\)\%\((l)\), x\_\(i, 2\)\%\((l)\), \[CenterEllipsis], x\_\(i, n\_\(l\_i\)\)\%\((l)\))\)}], ",", " ", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", \(N\_l\), ",", \(l = 0\), ",", "1", ",", "\[CenterEllipsis]", ",", \(L - 1\)}], TraditionalForm]], "NumberedEquation", CellTags->"Ed:Change10"], Cell[TextData[{ "which permits us to define the intra-partition mutual information of \ partition ", Cell[BoxData[ \(TraditionalForm\`i\)]], " of layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], " as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox[\(intra, i\), FontSlant->"Italic"], \((l)\)], "=", "\[AlignmentMarker]", RowBox[{\(I[x\_\(i, 1\)\%\((l)\); x\_\(i, 2\)\%\((l)\); \[CenterEllipsis]; x\_\(i, n\_\(l\_i\)\)\%\((l)\)]\), "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontSlant->"Italic"]}]], \(i, 1\), \((l)\)], SubsuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontSlant->"Italic"]}]], \(i, 2\), \((l)\)], "\[CenterEllipsis]", " ", SubsuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontSlant->"Italic"]}]], \(i, n\_\(l\_i\)\), \((l)\)], RowBox[{\(P\_i\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l)\)], ")"}], RowBox[{"log", "[", FractionBox[ RowBox[{\(P\_i\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l)\)], ")"}], \(\(\(P\_\(i, 1\)\%\((l)\)\)( x\_\(i, 1\)\%\((l)\))\) \(\(P\_\(i, 2\)\%\((l)\)\)( x\_\(i, 2\)\%\((l)\))\) \[CenterEllipsis]\ \ \(\(P\_\(i, n\_\(l\_i\)\)\%\((l)\)\)(x\_\(i, n\_\(l\_i\)\)\%\((l)\))\)\)], "]"}], " ", "i"}]}], "=", "1"}]}]}], ",", "2", ",", "\[CenterEllipsis]", ",", \(N\_l\), ",", \(l = 0\), ",", "1", ",", "\[CenterEllipsis]", ",", \(L - 1\)}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->{"Ed:Change11", "Ed:Change12"}], Cell[TextData[{ "Finally the analogue of ", ButtonBox["equation", ButtonData:>"Eq:25", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:25"], ") may be written in the form" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l)\)], "=", RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], "+", RowBox[{\(\[Sum]\+\(i = 1\)\%\(N\_l\)\), RowBox[{ SubsuperscriptBox["I", StyleBox[\(intra, i\), FontSlant->"Italic"], \((l)\)], " ", "l"}]}]}], "=", "0"}]}], ",", "1", ",", "\[CenterEllipsis]", ",", \(L - 1\)}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:32"], Cell[TextData[{ ButtonBox["Equation", ButtonData:>"Eq:28", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:28"], ") and ", ButtonBox["equation", ButtonData:>"Eq:32", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:32"], ") may be applied to the analysis of ", ButtonBox["equation", ButtonData:>"Eq:16", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:16"], "). This is achieved by using ", ButtonBox["equation", ButtonData:>"Eq:32", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:32"], ") to decompose the total mutual information of layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], ", and then using ", ButtonBox["equation", ButtonData:>"Eq:28", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:28"], ") to relate (via an inequality) the inter-partition mutual information of \ layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], " to the total mutual information of layer ", Cell[BoxData[ \(TraditionalForm\`l + 1\)]], ". Because the partitioning scheme used here is the same as that used in ", ButtonBox["equation", ButtonData:>"Eq:16", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:16"], "), we may directly relate the two calculations." }], "Text", CellTags->"Ed:Change13"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Quality of maximum entropy estimate" }], "Section", CellTags->"Sect:4"], Cell[TextData[{ "We shall now define a measure of the quality of the maximum entropy \ estimate ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], TraditionalForm]]], " given in ", ButtonBox["equation", ButtonData:>"Eq:16", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:16"], "), and we shall relate the properties of this measure to the properties of \ mutual information derived in ", ButtonBox["\[Section]", ButtonData:>"Sect:3", ButtonStyle->"Hyperlink"], CounterBox["Section", "Sect:3"], "." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", "Relative entropy" }], "Subsection"], Cell["\<\ An appropriate measure of the quality of the maximum entropy estimate is the \ so-called relative entropy defined as\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{"G", "=", RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{"P", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], RowBox[{"log", "[", FractionBox[ RowBox[{"P", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], RowBox[{"Q", "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]], "]"}]}]}]}]}], TraditionalForm]], "NumberedEquation", SpanMaxSize->Infinity, CellTags->"Eq:33"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], TraditionalForm]]], " is given in ", ButtonBox["equation", ButtonData:>"Eq:16", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:16"], "). ", Cell[BoxData[ FormBox[ RowBox[{"G", "=", RowBox[{ RowBox[{"0", " ", StyleBox["iff", FontSlant->"Italic"], " ", RowBox[{"P", "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}]}], "=", RowBox[{ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], " ", RowBox[{"\[ForAll]", " ", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)]}]}]}]}], TraditionalForm]]], ". ", Cell[BoxData[ \(TraditionalForm\`G\)]], " has the following interpretation in terms of generalised Bernoulli \ trials. Suppose a PDF ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], TraditionalForm]]], " is defined, and suppose that a large number ", Cell[BoxData[ \(TraditionalForm\`M\)]], " of samples ", Cell[BoxData[ FormBox[ SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], TraditionalForm]]], " are drawn independently from ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], TraditionalForm]]], ". Defining the observed PDF of the samples as ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], TraditionalForm]]], " we obtain ", Cell[BoxData[ \(TraditionalForm\`e\^\(M\ G\)\)]], " as the probability that ", Cell[BoxData[ \(TraditionalForm\`M\)]], " trials have the observed distribution of outcomes. Thus the smaller ", Cell[BoxData[ \(TraditionalForm\`e\^\(M\ G\)\)]], " is (or, equivalently, the more negative ", Cell[BoxData[ \(TraditionalForm\`G\)]], " is) the more surprised we are that ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], TraditionalForm]]], " is genuinely the distribution of outcomes of independent samples from ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["P", StyleBox["mem", FontSlant->"Italic"]], "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], TraditionalForm]]], "." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", "Relationship to mutual information" }], "Subsection"], Cell[TextData[{ ButtonBox["Equation", ButtonData:>"Eq:33", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:33"], ") can be evaluated for various configurations of the hierarchical tree in \ ", ButtonBox["equation", ButtonData:>"Eq:16", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:16"], "). Most simply we can set ", Cell[BoxData[ \(TraditionalForm\`L = 1\)]], " to obtain" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"G", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{ RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ SuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"]}]], \((0)\)], RowBox[{"P", "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], RowBox[{"log", "[", FractionBox[ RowBox[{"P", "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], RowBox[{ RowBox[{\(P\_1\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((0)\)], ")"}], RowBox[{\(P\_2\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((0)\)], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(P\_\(N\_0\)\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_0\), \((0)\)], ")"}]}]], "]"}]}]}]}], "+", RowBox[{"\[Integral]", RowBox[{ SuperscriptBox[ StyleBox[ RowBox[{"d", StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"]}]], \((0)\)], RowBox[{"P", "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold", FontSlant->"Plain"], \((0)\)], ")"}], \(log[\(\(p\_\(12 \[CenterEllipsis]\ \ N\_0\)\%\((1)\)\)(y\_1\%\((1)\), y\_2\%\((1)\), \[CenterEllipsis], \ y\_\(N\_0\)\%\((1)\))\)\/\(\(\(p\_1\%\((1)\)\)(y\_1\%\((1)\))\) \ \(\(p\_2\%\((1)\)\)(y\_2\%\((1)\))\) \[CenterEllipsis]\ \ \(\(p\_\(N\_0\)\%\((1)\)\)(y\_\(N\_0\)\%\((1)\))\)\)]\)}]}]}], "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((1)\)], "-", SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((0)\)]}], "\[LessEqual]", "0"}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->{"Ed:Change14", "Eq:34"}], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(y\_i\%\((1)\)\), "=", RowBox[{\(y\_i\%\((1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((0)\)], ")"}]}], ",", \(i = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", \(N\_0\)}], TraditionalForm]]], " is to be understood. ", ButtonBox["Equation", ButtonData:>"Eq:26", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:26"], ") for ", Cell[BoxData[ \(TraditionalForm\`l = 1\)]], " and ", ButtonBox["equation", ButtonData:>"Eq:27", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:27"], ") for ", Cell[BoxData[ \(TraditionalForm\`l = 0\)]], " have been used to express ", Cell[BoxData[ \(TraditionalForm\`G\)]], " in terms of mutual information, and the final inequality follows from ", ButtonBox["equation", ButtonData:>"Eq:28", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:28"], ") for ", Cell[BoxData[ \(TraditionalForm\`l = 0\)]], "." }], "Text"], Cell[TextData[{ "Similarly ", ButtonBox["equation", ButtonData:>"Eq:34", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:34"], ") can be generalised to arbitrary ", Cell[BoxData[ \(TraditionalForm\`L \[GreaterEqual] 1\)]], ", resulting in" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"G", "=", RowBox[{ RowBox[{\(\[Sum]\+\(l = 0\)\%\(L - 1\)\), RowBox[{"(", RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l + 1)\)], "-", SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)]}], ")"}]}], "\[LessEqual]", "0"}]}], TraditionalForm]], "NumberedEquation", CellTags->{"Ed:Change15", "Eq:35"}], Cell[TextData[{ "We see that the sum of the mutual information losses in passing from layer \ to layer determines ", Cell[BoxData[ \(TraditionalForm\`G\)]], ", and hence the quality of the maximum entropy estimate. We may express ", ButtonBox["equation", ButtonData:>"Eq:35", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:35"], ") in an alternative form by using ", ButtonBox["equation", ButtonData:>"Eq:32", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:32"], ") to yield" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"G", "=", RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((L)\)], "-", SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((0)\)], "+", RowBox[{\(\[Sum]\+\(l = 1\)\%\(L - 1\)\), RowBox[{\(\[Sum]\+\(i = 1\)\%\(N\_l\)\), SubsuperscriptBox["I", StyleBox[\(intra, i\), FontSlant->"Italic"], \((l)\)]}]}]}]}], TraditionalForm]], "NumberedEquation"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " ", "Conclusions" }], "Section"], Cell["\<\ The theory of hierarchical cluster decomposition of PDFs is not yet complete. \ However, in this note we have shown how that one can derive a remarkable \ number of useful results using standard (information theoretic) techniques.\ \>", "Text"], Cell["\<\ A major limitation of our theory is its limitation to tree-like hierarchies: \ there is only one path from each leaf node to the root. We have not yet found \ a way of deriving our maximum entropy results when this constraint is \ removed. This is a pity, because a tree structure imposes a somewhat \ artificial structure on the cluster decomposition of a PDF. We would prefer \ to have \"fuzzy\" boundaries between the clusters.\ \>", "Text"], Cell["\<\ We have not dwelt on the problem of designing the layer to layer mappings in \ our hierarchical network. This is the focus of a lot of our research \ activity, and our current best solution is to use so-called \"topographic \ mappings\". However, these leave a lot to be desired because:\ \>", "Text"], Cell["\<\ \t1. They violate the pure information theoretic approach of the rest of our \ theory. Our current best theory of topographic mappings is based on a minimum \ mean square error cost function approach (i.e. nothing to do with entropies, \ etc).\ \>", "Text", CellTags->"Ed:Change16"], Cell["\<\ \t2. They are designed using a pure \"unsupervised\" training algorithm, \ which limits the usefulness of the hierarchy of mappings to constructing \ cluster decompositions of PDFs. It would be very nice to \"kill two birds \ with one stone\" by forcing the network to compute otherwise useful \ quantities with its mappings.\ \>", "Text"], Cell["\<\ However, we find that using topographic mappings is a good holding strategy \ until something better comes along.\ \>", "Text"], Cell["\<\ A very important property of our cluster decomposition method (that is not \ widely advertised) is that it reduces to WISARD when only one layer is used. \ Where WISARD uses \"n-tuples\", we use \"n-tuples\" and \"n-tuples of \ n-tuples\" and so on. Thus we do not discard the \"inter-n-tuple correlations\ \", whereas WISARD does.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell["References", "Section"], Cell[TextData[{ "[1] Jeffreys H. ", StyleBox["Theory of Probability", FontSlant->"Italic"], ". Clarendon Press, 1939." }], "Reference", CellTags->"Ref:Jeffreys1939"], Cell[TextData[{ "[2] Cox R T. Probability, frequency and reasonable expectation. ", StyleBox["Am. J. Phys.", FontSlant->"Italic"], ", ", StyleBox["14", FontWeight->"Bold"], ", 1-13, 1946." }], "Reference", CellTags->"Ref:Cox1946"], Cell[TextData[{ "[3] Ackley D H, Hinton G E and Sejnowski T J. A learning algorithm for \ Boltzmann machines. ", StyleBox["Cogn. Sci.", FontSlant->"Italic"], ", ", StyleBox["9", FontWeight->"Bold"], ", 147-169, 1985." }], "Reference", CellTags->"Ref:AckleyHintonSejnowski1985"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/3815/3815.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "[4] Luttrell S P. The implications of Boltzmann-type machines for SAR data \ processing: a preliminary survey. Technical Report 3815, RSRE, 1985." }], "Reference", CellTags->"Ref:Luttrell1985"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/sp4_99/sp4_99.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "[5] Luttrell S P. The Gibbs Machine applied to hidden Markov model \ problems. Part 1: Basic theory. Technical Report SP4/99, RSRE, 1989." }], "Reference", CellTags->"Ref:Luttrell1989a"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/maxent88/maxent88.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "[6] Luttrell S P. The use of Bayesian and entropic methods in neural \ network theory. In Skilling J, editor, ", StyleBox["Maximum entropy and Bayesian methods", FontSlant->"Italic"], ", pages 363-370, Cambridge, 1989. Kluwer Academic Publishers." }], "Reference", CellTags->"Ref:Luttrell1989b"] }, Closed]] }, Open ]] }, FrontEndVersion->"5.0 for Microsoft Windows", ScreenRectangle->{{0, 1280}, {0, 941}}, WindowToolbars->{}, WindowSize->{665.375, 641}, WindowMargins->{{307.25, Automatic}, {Automatic, 50}}, Magnification->1, StyleDefinitions -> "Report.nb" ] (******************************************************************* Cached data follows. If you edit this Notebook file directly, not using Mathematica, you must remove the line containing CacheID at the top of the file. The cache data will then be recreated when you save this file from within Mathematica. *******************************************************************) (*CellTagsOutline CellTagsIndex->{ "Sect:2"->{ Cell[13580, 468, 113, 6, 70, "Section", CellTags->"Sect:2"]}, "Eq:1"->{ Cell[14199, 494, 577, 18, 70, "NumberedEquation", CellTags->"Eq:1"]}, "Sect:2.1"->{ Cell[17004, 597, 147, 9, 70, "Subsection", CellTags->"Sect:2.1"]}, "Eq:2"->{ Cell[18172, 644, 1545, 46, 70, "NumberedEquation", CellTags->"Eq:2"]}, "Eq:3"->{ Cell[20024, 704, 406, 12, 70, "NumberedEquation", CellTags->"Eq:3"]}, "Eq:4"->{ Cell[21159, 743, 1311, 36, 70, "NumberedEquation", CellTags->"Eq:4"]}, "Eq:5"->{ Cell[22689, 792, 817, 22, 70, "NumberedEquation", CellTags->"Eq:5"]}, "Eq:6"->{ Cell[24145, 841, 1148, 33, 70, "NumberedEquation", CellTags->"Eq:6"]}, "Ed:Change1"->{ Cell[26083, 903, 3246, 80, 70, "NumberedEquation", CellTags->{"Ed:Change1", "Eq:7"}]}, "Eq:7"->{ Cell[26083, 903, 3246, 80, 70, "NumberedEquation", CellTags->{"Ed:Change1", "Eq:7"}]}, "Ed:Change2"->{ Cell[30201, 1015, 1802, 51, 70, "NumberedEquation", CellTags->{"Ed:Change2", "Eq:8"}]}, "Eq:8"->{ Cell[30201, 1015, 1802, 51, 70, "NumberedEquation", CellTags->{"Ed:Change2", "Eq:8"}]}, "Ed:Change3"->{ Cell[32006, 1068, 1456, 46, 70, "Text", CellTags->"Ed:Change3"]}, "Eq:9"->{ Cell[33465, 1116, 2785, 75, 70, "NumberedEquation", CellTags->"Eq:9"]}, "Eq:10"->{ Cell[39079, 1292, 1757, 51, 70, "NumberedEquation", CellTags->"Eq:10"]}, "Ed:Change4"->{ Cell[40839, 1345, 1572, 54, 70, "Text", CellTags->"Ed:Change4"]}, "Sect:2.2"->{ Cell[42448, 1404, 248, 13, 70, "Subsection", CellTags->"Sect:2.2"]}, "Eq:11"->{ Cell[44107, 1481, 1375, 39, 70, "NumberedEquation", CellTags->"Eq:11"]}, "Ed:Change5"->{ Cell[45874, 1535, 1679, 44, 70, "NumberedEquation", CellTags->"Ed:Change5"]}, "Eq:14"->{ Cell[47556, 1581, 1218, 33, 70, "NumberedEquation", CellTags->"Eq:14"]}, "Eq:15"->{ Cell[51240, 1715, 2184, 58, 70, "NumberedEquation", CellTags->"Eq:15"]}, "Eq:16"->{ Cell[59617, 1993, 3835, 92, 70, "NumberedEquation", CellTags->{"Eq:16", "Ed:Change7"}]}, "Ed:Change7"->{ Cell[59617, 1993, 3835, 92, 70, "NumberedEquation", CellTags->{"Eq:16", "Ed:Change7"}]}, "Eq:17"->{ Cell[63455, 2087, 806, 22, 70, "NumberedEquation", CellTags->"Eq:17"]}, "Eq:19"->{ Cell[64721, 2123, 451, 12, 70, "NumberedEquation", CellTags->"Eq:19"]}, "Sect:3"->{ Cell[66809, 2201, 110, 6, 70, "Section", CellTags->"Sect:3"]}, "Eq:20"->{ Cell[68353, 2267, 115, 3, 70, "NumberedEquation", CellTags->"Eq:20"]}, "Eq:21"->{ Cell[68471, 2272, 129, 3, 70, "NumberedEquation", CellTags->"Eq:21"]}, "Eq:22"->{ Cell[68603, 2277, 129, 3, 70, "NumberedEquation", CellTags->"Eq:22"]}, "Eq:23"->{ Cell[72261, 2413, 210, 4, 70, "NumberedEquation", CellTags->"Eq:23"]}, "Eq:24"->{ Cell[73285, 2448, 1939, 55, 70, "NumberedEquation", CellTags->"Eq:24"]}, "Ed:Change6"->{ Cell[75227, 2505, 811, 21, 70, "NumberedEquation", CellTags->{"Ed:Change6", "Eq:25"}]}, "Eq:25"->{ Cell[75227, 2505, 811, 21, 70, "NumberedEquation", CellTags->{"Ed:Change6", "Eq:25"}]}, "Ed:Change8"->{ Cell[78974, 2642, 1913, 42, 70, "NumberedEquation", CellTags->{"Ed:Change8", "Eq:26"}]}, "Eq:26"->{ Cell[78974, 2642, 1913, 42, 70, "NumberedEquation", CellTags->{"Ed:Change8", "Eq:26"}]}, "Ed:Change9"->{ Cell[81120, 2697, 3094, 71, 70, "NumberedEquation", CellTags->{"Ed:Change9", "Eq:27"}]}, "Eq:27"->{ Cell[81120, 2697, 3094, 71, 70, "NumberedEquation", CellTags->{"Ed:Change9", "Eq:27"}]}, "Eq:28"->{ Cell[85158, 2807, 481, 12, 70, "NumberedEquation", CellTags->"Eq:28"]}, "Ed:Change10"->{ Cell[88497, 2918, 541, 13, 70, "NumberedEquation", CellTags->"Ed:Change10"]}, "Ed:Change11"->{ Cell[89278, 2944, 2334, 52, 70, "NumberedEquation", CellTags->{"Ed:Change11", "Ed:Change12"}]}, "Ed:Change12"->{ Cell[89278, 2944, 2334, 52, 70, "NumberedEquation", CellTags->{"Ed:Change11", "Ed:Change12"}]}, "Eq:32"->{ Cell[91842, 3009, 760, 20, 70, "NumberedEquation", CellTags->"Eq:32"]}, "Ed:Change13"->{ Cell[92605, 3031, 1443, 54, 70, "Text", CellTags->"Ed:Change13"]}, "Sect:4"->{ Cell[94097, 3091, 121, 5, 70, "Section", CellTags->"Sect:4"]}, "Eq:33"->{ Cell[95553, 3153, 1621, 44, 70, "NumberedEquation", CellTags->"Eq:33"]}, "Ed:Change14"->{ Cell[101686, 3358, 3074, 70, 70, "NumberedEquation", CellTags->{"Ed:Change14", "Eq:34"}]}, "Eq:34"->{ Cell[101686, 3358, 3074, 70, 70, "NumberedEquation", CellTags->{"Ed:Change14", "Eq:34"}]}, "Ed:Change15"->{ Cell[106278, 3492, 560, 14, 70, "NumberedEquation", CellTags->{"Ed:Change15", "Eq:35"}]}, "Eq:35"->{ Cell[106278, 3492, 560, 14, 70, "NumberedEquation", CellTags->{"Ed:Change15", "Eq:35"}]}, "Ed:Change16"->{ Cell[109172, 3580, 294, 6, 70, "Text", CellTags->"Ed:Change16"]}, "Ref:Jeffreys1939"->{ Cell[110385, 3614, 177, 6, 70, "Reference", CellTags->"Ref:Jeffreys1939"]}, "Ref:Cox1946"->{ Cell[110565, 3622, 250, 9, 70, "Reference", CellTags->"Ref:Cox1946"]}, "Ref:AckleyHintonSejnowski1985"->{ Cell[110818, 3633, 297, 10, 70, "Reference", CellTags->"Ref:AckleyHintonSejnowski1985"]}, "Ref:Luttrell1985"->{ Cell[111118, 3645, 422, 13, 70, "Reference", CellTags->"Ref:Luttrell1985"]}, "Ref:Luttrell1989a"->{ Cell[111543, 3660, 419, 13, 70, "Reference", CellTags->"Ref:Luttrell1989a"]}, "Ref:Luttrell1989b"->{ Cell[111965, 3675, 547, 17, 70, "Reference", CellTags->"Ref:Luttrell1989b"]} } *) (*CellTagsIndex CellTagsIndex->{ {"Sect:2", 113211, 3714}, {"Eq:1", 113295, 3717}, {"Sect:2.1", 113391, 3720}, {"Eq:2", 113480, 3723}, {"Eq:3", 113573, 3726}, {"Eq:4", 113665, 3729}, {"Eq:5", 113758, 3732}, {"Eq:6", 113850, 3735}, {"Ed:Change1", 113949, 3738}, {"Eq:7", 114058, 3741}, {"Ed:Change2", 114173, 3744}, {"Eq:8", 114283, 3747}, {"Ed:Change3", 114399, 3750}, {"Eq:9", 114487, 3753}, {"Eq:10", 114582, 3756}, {"Ed:Change4", 114683, 3759}, {"Sect:2.2", 114775, 3762}, {"Eq:11", 114867, 3765}, {"Ed:Change5", 114968, 3768}, {"Eq:14", 115069, 3771}, {"Eq:15", 115165, 3774}, {"Eq:16", 115261, 3777}, {"Ed:Change7", 115378, 3780}, {"Eq:17", 115490, 3783}, {"Eq:19", 115585, 3786}, {"Sect:3", 115681, 3789}, {"Eq:20", 115767, 3792}, {"Eq:21", 115861, 3795}, {"Eq:22", 115955, 3798}, {"Eq:23", 116049, 3801}, {"Eq:24", 116143, 3804}, {"Ed:Change6", 116244, 3807}, {"Eq:25", 116355, 3810}, {"Ed:Change8", 116471, 3813}, {"Eq:26", 116583, 3816}, {"Ed:Change9", 116700, 3819}, {"Eq:27", 116812, 3822}, {"Eq:28", 116924, 3825}, {"Ed:Change10", 117025, 3828}, {"Ed:Change11", 117132, 3831}, {"Ed:Change12", 117257, 3834}, {"Eq:32", 117376, 3837}, {"Ed:Change13", 117477, 3840}, {"Sect:4", 117568, 3843}, {"Eq:33", 117654, 3846}, {"Ed:Change14", 117756, 3849}, {"Eq:34", 117870, 3852}, {"Ed:Change15", 117990, 3855}, {"Eq:35", 118103, 3858}, {"Ed:Change16", 118222, 3861}, {"Ref:Jeffreys1939", 118322, 3864}, {"Ref:Cox1946", 118427, 3867}, {"Ref:AckleyHintonSejnowski1985", 118545, 3870}, {"Ref:Luttrell1985", 118669, 3873}, {"Ref:Luttrell1989a", 118781, 3876}, {"Ref:Luttrell1989b", 118894, 3879} } *) (*NotebookFileOutline Notebook[{ Cell[CellGroupData[{ Cell[1776, 53, 26, 0, 70, "Section 1"], Cell[CellGroupData[{ Cell[1827, 57, 39, 0, 70, "Subsection"], Cell[1869, 59, 497, 18, 70, "Text"], Cell[2369, 79, 1249, 44, 70, "Text"], Cell[3621, 125, 1540, 51, 70, "Text"], Cell[5164, 178, 1032, 34, 70, "Text"], Cell[6199, 214, 321, 11, 70, "Text"], Cell[6523, 227, 494, 16, 70, "Text"], Cell[7020, 245, 1068, 34, 70, "Text"], Cell[8091, 281, 264, 8, 70, "Text"], Cell[8358, 291, 259, 8, 70, "Text"], Cell[8620, 301, 1015, 32, 70, "Text"], Cell[9638, 335, 227, 8, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[9914, 349, 54, 3, 70, "Title"], Cell[9971, 354, 118, 5, 70, "Author"], Cell[10092, 361, 91, 2, 70, "Text"], Cell[10186, 365, 69, 0, 70, "Text"], Cell[10258, 367, 552, 11, 70, "Abstract"], Cell[CellGroupData[{ Cell[10835, 382, 78, 4, 70, "Section 1"], Cell[10916, 388, 1016, 28, 70, "Text"], Cell[11935, 418, 929, 27, 70, "Text"], Cell[12867, 447, 249, 7, 70, "Text"], Cell[13119, 456, 424, 7, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[13580, 468, 113, 6, 70, "Section", CellTags->"Sect:2"], Cell[13696, 476, 500, 16, 70, "Text"], Cell[14199, 494, 577, 18, 70, "NumberedEquation", CellTags->"Eq:1"], Cell[14779, 514, 1038, 33, 70, "Text"], Cell[15820, 549, 1159, 44, 70, "Text"], Cell[CellGroupData[{ Cell[17004, 597, 147, 9, 70, "Subsection", CellTags->"Sect:2.1"], Cell[17154, 608, 1015, 34, 70, "Text"], Cell[18172, 644, 1545, 46, 70, "NumberedEquation", CellTags->"Eq:2"], Cell[19720, 692, 301, 10, 70, "Text"], Cell[20024, 704, 406, 12, 70, "NumberedEquation", CellTags->"Eq:3"], Cell[20433, 718, 723, 23, 70, "Text"], Cell[21159, 743, 1311, 36, 70, "NumberedEquation", CellTags->"Eq:4"], Cell[22473, 781, 213, 9, 70, "Text"], Cell[22689, 792, 817, 22, 70, "NumberedEquation", CellTags->"Eq:5"], Cell[23509, 816, 585, 21, 70, "Text"], Cell[24097, 839, 45, 0, 70, "Text"], Cell[24145, 841, 1148, 33, 70, "NumberedEquation", CellTags->"Eq:6"], Cell[25296, 876, 784, 25, 70, "Text"], Cell[26083, 903, 3246, 80, 70, "NumberedEquation", CellTags->{"Ed:Change1", "Eq:7"}], Cell[29332, 985, 866, 28, 70, "Text"], Cell[30201, 1015, 1802, 51, 70, "NumberedEquation", CellTags->{"Ed:Change2", "Eq:8"}], Cell[32006, 1068, 1456, 46, 70, "Text", CellTags->"Ed:Change3"], Cell[33465, 1116, 2785, 75, 70, "NumberedEquation", CellTags->"Eq:9"], Cell[36253, 1193, 258, 9, 70, "Text"], Cell[36514, 1204, 2341, 75, 70, "Text"], Cell[38858, 1281, 218, 9, 70, "Text"], Cell[39079, 1292, 1757, 51, 70, "NumberedEquation", CellTags->"Eq:10"], Cell[40839, 1345, 1572, 54, 70, "Text", CellTags->"Ed:Change4"] }, Closed]], Cell[CellGroupData[{ Cell[42448, 1404, 248, 13, 70, "Subsection", CellTags->"Sect:2.2"], Cell[42699, 1419, 1405, 60, 70, "Text"], Cell[44107, 1481, 1375, 39, 70, "NumberedEquation", CellTags->"Eq:11"], Cell[45485, 1522, 386, 11, 70, "NumberedEquation"], Cell[45874, 1535, 1679, 44, 70, "NumberedEquation", CellTags->"Ed:Change5"], Cell[47556, 1581, 1218, 33, 70, "NumberedEquation", CellTags->"Eq:14"], Cell[48777, 1616, 365, 16, 70, "Text"], Cell[49145, 1634, 2092, 79, 70, "Text"], Cell[51240, 1715, 2184, 58, 70, "NumberedEquation", CellTags->"Eq:15"], Cell[53427, 1775, 510, 23, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[53974, 1803, 132, 8, 70, "Subsection"], Cell[54109, 1813, 2446, 79, 70, "Text"], Cell[56558, 1894, 1933, 66, 70, "Text"], Cell[58494, 1962, 967, 24, 70, "Text"], Cell[59464, 1988, 150, 3, 70, "Text"], Cell[59617, 1993, 3835, 92, 70, "NumberedEquation", CellTags->{"Eq:16", "Ed:Change7"}], Cell[63455, 2087, 806, 22, 70, "NumberedEquation", CellTags->"Eq:17"], Cell[64264, 2111, 454, 10, 70, "NumberedEquation"], Cell[64721, 2123, 451, 12, 70, "NumberedEquation", CellTags->"Eq:19"], Cell[65175, 2137, 1585, 58, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[66809, 2201, 110, 6, 70, "Section", CellTags->"Sect:3"], Cell[66922, 2209, 945, 31, 70, "Text"], Cell[CellGroupData[{ Cell[67892, 2244, 128, 8, 70, "Subsection"], Cell[68023, 2254, 327, 11, 70, "Text"], Cell[68353, 2267, 115, 3, 70, "NumberedEquation", CellTags->"Eq:20"], Cell[68471, 2272, 129, 3, 70, "NumberedEquation", CellTags->"Eq:21"], Cell[68603, 2277, 129, 3, 70, "NumberedEquation", CellTags->"Eq:22"], Cell[68735, 2282, 2348, 86, 70, "Text"], Cell[71086, 2370, 946, 28, 70, "Text"], Cell[72035, 2400, 223, 11, 70, "Text"], Cell[72261, 2413, 210, 4, 70, "NumberedEquation", CellTags->"Eq:23"], Cell[72474, 2419, 674, 22, 70, "Text"], Cell[73151, 2443, 131, 3, 70, "Text"], Cell[73285, 2448, 1939, 55, 70, "NumberedEquation", CellTags->"Eq:24"], Cell[75227, 2505, 811, 21, 70, "NumberedEquation", CellTags->{"Ed:Change6", "Eq:25"}], Cell[76041, 2528, 2037, 67, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[78115, 2600, 154, 8, 70, "Subsection"], Cell[78272, 2610, 699, 30, 70, "Text"], Cell[78974, 2642, 1913, 42, 70, "NumberedEquation", CellTags->{"Ed:Change8", "Eq:26"}], Cell[80890, 2686, 227, 9, 70, "Text"], Cell[81120, 2697, 3094, 71, 70, "NumberedEquation", CellTags->{"Ed:Change9", "Eq:27"}], Cell[84217, 2770, 882, 33, 70, "Text"], Cell[85102, 2805, 53, 0, 70, "Text"], Cell[85158, 2807, 481, 12, 70, "NumberedEquation", CellTags->"Eq:28"], Cell[85642, 2821, 1242, 35, 70, "Text"], Cell[86887, 2858, 233, 9, 70, "Text"], Cell[87123, 2869, 940, 26, 70, "NumberedEquation"], Cell[88066, 2897, 428, 19, 70, "Text"], Cell[88497, 2918, 541, 13, 70, "NumberedEquation", CellTags->"Ed:Change10"], Cell[89041, 2933, 234, 9, 70, "Text"], Cell[89278, 2944, 2334, 52, 70, "NumberedEquation", CellTags->{"Ed:Change11", "Ed:Change12"}], Cell[91615, 2998, 224, 9, 70, "Text"], Cell[91842, 3009, 760, 20, 70, "NumberedEquation", CellTags->"Eq:32"], Cell[92605, 3031, 1443, 54, 70, "Text", CellTags->"Ed:Change13"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[94097, 3091, 121, 5, 70, "Section", CellTags->"Sect:4"],