(************** Content-type: application/mathematica ************** CreatedBy='Mathematica 5.0' Mathematica-Compatible Notebook This notebook can be used with any Mathematica-compatible application, such as Mathematica, MathReader or Publicon. The data for the notebook starts with the line containing stars above. To get the notebook into a Mathematica-compatible application, do one of the following: * Save the data starting with the line of stars above into a file with a name ending in .nb, then open the file inside the application; * Copy the data starting with the line of stars above to the clipboard, then use the Paste menu command inside the application. Data for notebooks contains only printable 7-bit ASCII and can be sent directly in email or through ftp in text mode. Newlines can be CR, LF or CRLF (Unix, Macintosh or MS-DOS style). NOTE: If you modify the data for this notebook not in a Mathematica- compatible application, you must delete the line below containing the word CacheID, otherwise Mathematica-compatible applications may try to use invalid cache data. For more information on notebooks and Mathematica-compatible applications, contact Wolfram Research: web: http://www.wolfram.com email: info@wolfram.com phone: +1-217-398-0700 (U.S.) Notebook reader applications are available free of charge from Wolfram Research. *******************************************************************) (*CacheID: 232*) (*NotebookFileLineBreakTest NotebookFileLineBreakTest*) (*NotebookOptionsPosition[ 117494, 3782]*) (*NotebookOutlinePosition[ 126253, 4028]*) (* CellTagsIndexPosition[ 124283, 3968]*) (*WindowFrame->Normal*) Notebook[{ Cell[CellGroupData[{ Cell["Notes", "Section 1"], Cell[CellGroupData[{ Cell["Editorial Changes", "Subsection"], Cell[TextData[{ "Citations \"", Cell[BoxData[ \(TraditionalForm\`\[CenterEllipsis]\^m\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`\[CenterEllipsis]\ [m]\)]], "\"." }], "Text"], Cell["\<\ Equations reformatted so that multiple equations on a single line are \ converted to single equations on multiple lines.\ \>", "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change1", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(p\_12\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], "/", RowBox[{"(", RowBox[{ RowBox[{\(p\_1\), "(", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ")"}], RowBox[{\(p\_2\), "(", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], ")"}]}], ")"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{\(p\_12\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], RowBox[{ RowBox[{\(p\_1\), "(", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ")"}], RowBox[{\(p\_2\), "(", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], ")"}]}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:entropystatn", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{\(p\_\(1, 2, \[CenterEllipsis], N\)\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ",", "\[CenterEllipsis]", ",", RowBox[{\(y\_N\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}]}], ")"}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{\(p\_\(12 \[CenterEllipsis]\ N\)\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ",", "\[CenterEllipsis]", ",", RowBox[{\(y\_N\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}]}], ")"}], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Eq:relativeentropy", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"Q", "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change2", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"fast\" changed to \"fact\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change3a", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change3b", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`I[X; Y]\)]], "\" written on left hand side of equation, in order to make a group of 3 \ related equations line up nicely." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change4", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox["dy", FontSlant->"Italic"], "1", \((l)\)], SubsuperscriptBox[ StyleBox["dy", FontSlant->"Italic"], "2", \((l)\)], "\[CenterEllipsis]", " ", SubsuperscriptBox[ StyleBox["y", FontSlant->"Italic"], \(N\_\(l - 1\)\), \((l)\)]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox["dy", FontSlant->"Italic"], "1", \((l)\)], SubsuperscriptBox[ StyleBox["dy", FontSlant->"Italic"], "2", \((l)\)], "\[CenterEllipsis]", " ", SubsuperscriptBox[ StyleBox["dy", FontSlant->"Italic"], \(N\_\(l - 1\)\), \((l)\)]}]}], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change5", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`I[X\_\(p\_i\); X\_\(p\_i + 1\); \[CenterEllipsis]\ ; X\_\(p\_\(i + 1\)\)]\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`I[X\_\(p\_i + 1\); X\_\(p\_i + 2\); \[CenterEllipsis]\ ; X\_\(p\_\(i + 1\)\)]\)]], "\"." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["A hierarchical network for clutter and texture modelling", "Title"], Cell["\<\ Stephen P Luttrell Defence Research Agency: Electronics Division St Andrews Rd, Malvern, WORCS, WR14 3PS, United Kingdom Internet: luttrell%hermes.mod.uk@relay.mod.uk\ \>", "Author"], Cell["\<\ This appeared in Proceedings of the SPIE Conference on Adaptive Signal \ Processing, San Diego, 518-528, 1991.\ \>", "Text"], Cell["\<\ ABSTRACT The presence of clutter complicates the location of targets in time series \ and images. Various types of adaptive clutter model have been proposed to \ deal with this problem. In this paper we treat clutter as a type of texture, \ and we propose a novel type of hierarchical Gibbs distribution texture model. \ To optimise this type of model, we define a relative entropy cost function \ which we decompose into a sum over a number of terms, each of which can be \ interpreted as the mutual information between clusters of samples of the \ data. Furthermore we show how the various terms of this cost function can be \ used to construct an image-like representation of the relative entropy. \ Finally, using a Brodatz texture image, we present an example of this type of \ decomposition, and demonstrate that a statistical anomaly in the Brodatz \ texture image can be easily located.\ \>", "Abstract"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". INTRODUCTION" }], "Section 1"], Cell[TextData[{ "This paper addresses the problem of building a Gibbs distribution (i.e. \ maximum entropy) model approximating the joint probability density function \ (PDF) of the pixels of a textured image. Such a model can be used in Bayesian \ ", Cell[BoxData[ \(TraditionalForm\`inference\)]], " [", ButtonBox["1", ButtonData:>"Ref:Jeffreys1939", ButtonStyle->"Hyperlink"], ", ", ButtonBox["2", ButtonData:>"Ref:Cox1946", ButtonStyle->"Hyperlink"], "] to determine the probability that each area of the image is a piece of \ texture or something entirely different (e.g. an anomaly). There are many \ approaches to constructing Gibbs distribution models, such as the Boltzmann \ machine [", ButtonBox["3", ButtonData:>"Ref:AckleyHintonSejnowski1985", ButtonStyle->"Hyperlink"], "] or the Gibbs machine [", ButtonBox["4", ButtonData:>"Ref:Luttrell1985", ButtonStyle->"Hyperlink"], ", ", ButtonBox["5", ButtonData:>"Ref:Luttrell1989", ButtonStyle->"Hyperlink"], "]. In this paper we use a Gibbs distribution model [", ButtonBox["6", ButtonData:>"Ref:Luttrell1991", ButtonStyle->"Hyperlink"], "], which is built up from the statistical properties measured at each \ layer of a hierarchy of transformations of the input data. The goal of this \ paper is not the training of, but rather the interpretation of, this type of \ hierarchical Gibbs distribution model." }], "Text"], Cell[TextData[{ "In ", ButtonBox["section", ButtonData:>"Sect:2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2"], " we derive the basic maximum entropy expressions for a Gibbs distribution, \ constrained so that various of its marginal PDFs adopt required values. In ", ButtonBox["section", ButtonData:>"Sect:3", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:3"], " we introduce a relative entropy cost function in order to quantify the \ performance of the Gibbs distribution in ", ButtonBox["section", ButtonData:>"Sect:2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2"], ", and we show how this relative entropy may be expressed as a sum of \ mutual informations. In ", ButtonBox["section", ButtonData:>"Sect:4", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:4"], " we present an application of the relative entropy results in ", ButtonBox["section", ButtonData:>"Sect:3", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:3"], " to a Brodatz texture image [", ButtonBox["7", ButtonData:>"Ref:Brodatz1966", ButtonStyle->"Hyperlink"], "], which we display as a relative entropy image." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". CLUSTER DECOMPOSITION" }], "Section", CellTags->"Sect:2"], Cell[TextData[{ "In this section we derive a Gibbs distribution model of the PDF ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " of a multidimensional data vector ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold"], TraditionalForm]]], ". The particular Gibbs distribution is determined by the set of \ constraints that we impose: we choose to constrain some of its marginal \ PDFs." }], "Text"], Cell[TextData[{ "We present a simplified derivation in which ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold"], "=", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}], TraditionalForm]]], ", which we then generalise to the case ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold"], "=", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"]}], ")"}]}], TraditionalForm]]], " Finally, we further generalise this to accommodate a hierarchical \ decomposition of the joint PDF." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Two clusters" }], "Subsection", CellTags->"Sect:2.1"], Cell[TextData[{ "Let ", Cell[BoxData[ \(TraditionalForm\`N = 2\)]], ", whence ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold"], "=", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], "=", RowBox[{"P", "(", StyleBox[ RowBox[{ SubscriptBox["x", StyleBox["1", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"]]}], FontWeight->"Bold"], ")"}]}], TraditionalForm]]], ". Define two marginals as" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "\[Congruent]", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{\(P\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "\[Congruent]", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], RowBox[{\(P\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:margpdf2"], Cell[TextData[{ "and derive transformed versions of the ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], TraditionalForm]]], " as" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{\(y\_1\), "=", "\[AlignmentMarker]", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], TraditionalForm], "\n", FormBox[ RowBox[{\(y\_2\), "=", "\[AlignmentMarker]", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:map2"], Cell[TextData[{ "and use ", ButtonBox["equation", ButtonData:>"Eq:map2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:map2"], ") to obtain the joint PDF ", Cell[BoxData[ \(TraditionalForm\`\(p\_12\)(y\_1, y\_2)\)]], " of the ", Cell[BoxData[ \(TraditionalForm\`y\_i\)]], " as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{\(\(p\_12\)(y\_1, y\_2)\), "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{\(P\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{\(y\_1\), "-", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{\(y\_2\), "-", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:pdfmap2"], Cell[TextData[{ "where the Dirac delta functions constrain the integration to the surface \ ", Cell[BoxData[ FormBox[ RowBox[{\((y\_1, y\_2)\), "=", RowBox[{"(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}]}], TraditionalForm]]], ". From ", ButtonBox["equation", ButtonData:>"Eq:pdfmap2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:pdfmap2"], ") we define two further marginals as" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{\(\(p\_1\) \((y\_1)\)\), "\[Congruent]", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox["dy", FontSlant->"Italic"], "2"], \(\(p\_12\)(y\_1, y\_2)\)}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{\(\(p\_2\) \((y\_2)\)\), "\[Congruent]", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox["dy", FontSlant->"Italic"], "1"], \(\(p\_12\)(y\_1, y\_2)\)}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:margpdfmap2"], Cell[TextData[{ "We use ", Cell[BoxData[ FormBox[ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], TraditionalForm]]], ", ", Cell[BoxData[ FormBox[ RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ \(TraditionalForm\`\(p\_12\)(y\_1, y\_2)\)]], " as constraints on a maximum entropy estimate ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", RowBox[{ StyleBox["mem", FontSlant->"Italic"], ",", "12"}]], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox[ RowBox[{ SubscriptBox["x", StyleBox["1", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"]]}], FontWeight->"Bold"], ")"}], TraditionalForm]]], ". We present this derivation in detail in the appendix in ", ButtonBox["section", ButtonData:>"Sect:6.1", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:6.1"], ".", CounterBox["Subsection", "Sect:6.1"], ", and the final result is" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox["Q", RowBox[{ StyleBox["mem", FontSlant->"Italic"], ",", "12"}]], "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], "=", RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], FractionBox[ RowBox[{\(p\_12\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], RowBox[{ RowBox[{\(p\_1\), "(", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ")"}], RowBox[{\(p\_2\), "(", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], ")"}]}]]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:entropystat2bis"], Cell[TextData[{ "This result has an intuitively obvious form. The ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], TraditionalForm]]], " factor is the solution that we would expect if the constraints on ", Cell[BoxData[ \(TraditionalForm\`\(p\_12\)(y\_1, y\_2)\)]], " were omitted. This is because there would then be no information about \ correlations between ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], ". The dimensionless factor ", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{\(p\_12\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], RowBox[{ RowBox[{\(p\_1\), "(", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ")"}], RowBox[{\(p\_2\), "(", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], ")"}]}]], TraditionalForm]]], " introduces knowledge of correlations between ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], ". Note that this factor is unity when ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], TraditionalForm]]], " are statistically independent variables." }], "Text", CellTags->"Ed:Change1"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", Cell[BoxData[ FormBox[ StyleBox["N", FontWeight->"Plain"], TraditionalForm]]], " clusters" }], "Subsection"], Cell[TextData[{ "We now extend the results of ", ButtonBox["section", ButtonData:>"Sect:2.1", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2.1"], ".", CounterBox["Subsection", "Sect:2.1"], " to ", Cell[BoxData[ \(TraditionalForm\`N\)]], " clusters. Thus" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold"], "=", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"]}], ")"}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:partition"], Cell[TextData[{ "No further derivations are needed because the details are almost identical \ to those in ", ButtonBox["section", ButtonData:>"Sect:2.1", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2.1"], ".", CounterBox["Subsection", "Sect:2.1"], ". The results which correspond to ", ButtonBox["equation", ButtonData:>"Eq:margpdf2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:margpdf2"], "), ", ButtonBox["equation", ButtonData:>"Eq:map2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:map2"], "), ", ButtonBox["equation", ButtonData:>"Eq:pdfmap2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:pdfmap2"], ") and ", ButtonBox["equation", ButtonData:>"Eq:margpdfmap2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:margpdfmap2"], ") are respectively (for ", Cell[BoxData[ \(TraditionalForm\`i = 1, 2, \[CenterEllipsis], N\)]], ")" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_i\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]}], "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], "\[CenterEllipsis]", " ", OverscriptBox[ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "i"], "^"], "\[CenterEllipsis]", " ", SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], \(N - 1\)], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "N"], RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:margpdfn"], Cell[BoxData[ FormBox[ RowBox[{\(y\_i\), "=", RowBox[{\(y\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:mapn"], Cell[BoxData[ FormBox[ RowBox[{\(\(p\_\(12 \[CenterEllipsis]\ N\)\)(y\_1, y\_2, \[CenterEllipsis], y\_N)\), "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], "\[CenterEllipsis]", " ", SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "N"], RowBox[{"\[Delta]", "(", RowBox[{\(y\_1\), "-", RowBox[{\(y\_1\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{\(y\_2\), "-", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], "\[CenterEllipsis]", " ", RowBox[{"\[Delta]", "(", RowBox[{\(y\_N\), "-", RowBox[{\(y\_N\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}]}]}], ")"}], RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:pdfmapn"], Cell[BoxData[ FormBox[ RowBox[{\(\(p\_i\)(y\_i)\), "=", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox["dy", FontSlant->"Italic"], "1"], SubscriptBox[ StyleBox["dy", FontSlant->"Italic"], "2"], "\[CenterEllipsis]", " ", OverscriptBox[ SubscriptBox[ StyleBox["dy", FontSlant->"Italic"], "i"], "^"], "\[CenterEllipsis]", " ", SubscriptBox[ StyleBox["dy", FontSlant->"Italic"], \(N - 1\)], SubscriptBox[ StyleBox["dy", FontSlant->"Italic"], "N"], \(\(p\_\(12 \[CenterEllipsis]\ N\)\)(y\_1, y\_2, \[CenterEllipsis], y\_N)\)}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:margpdfmapn"], Cell[TextData[{ "where the hats in ", ButtonBox["equation", ButtonData:>"Eq:margpdfn", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:margpdfn"], ") and ", ButtonBox["equation", ButtonData:>"Eq:margpdfmapn", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:margpdfmapn"], ") denote an omitted integral. This leads to a generalisation of ", ButtonBox["equation", ButtonData:>"Eq:entropystat2bis", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:entropystat2bis"], ") as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], "=", RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(P\_N\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}], FractionBox[ RowBox[{\(p\_\(12 \[CenterEllipsis]\ N\)\), RowBox[{"(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ",", "\[CenterEllipsis]", ",", RowBox[{\(y\_N\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}]}], ")"}]}], RowBox[{ RowBox[{\(p\_1\), "(", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ")"}], \(p\_2\), RowBox[{"(", RowBox[{\(y\_2\), RowBox[{"(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(p\_N\), "(", RowBox[{\(y\_N\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], ")"}], ")"}]}]]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:entropystatn"], Cell[TextData[{ "The interpretation of ", ButtonBox["equation", ButtonData:>"Eq:entropystatn", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:entropystatn"], ") is analogous that of ", ButtonBox["equation", ButtonData:>"Eq:entropystat2bis", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:entropystat2bis"], ")." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Hierarchical clusters" }], "Subsection"], Cell[TextData[{ "First of all, we extend our notation in ", ButtonBox["equation", ButtonData:>"Eq:map2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:map2"], ") to accommodate an arbitrary number of stages of transformation. Thus we \ write " }], "Text"], Cell[BoxData[ FormBox[GridBox[{ { SuperscriptBox[ StyleBox["X", FontWeight->"Bold"], \((l)\)], "=", RowBox[{"\[AlignmentMarker]", RowBox[{"(", RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((l)\)], ",", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((l)\)], ",", "\[CenterEllipsis]", ",", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_l\), \((l)\)]}], ")"}]}], " ", \(l = 0, 1, \[CenterEllipsis], L - 1\), " ", " "}, {\(y\_i\%\((l)\)\), "=", RowBox[{\(y\_i\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l - 1)\)], ")"}], " ", \(l = 1, 2, \[CenterEllipsis], L\), " ", \(i = 1, 2, \[CenterEllipsis], N\_\(l - 1\)\)}, { SuperscriptBox[ StyleBox["X", FontWeight->"Bold"], \((l)\)], "=", \((y\_1\%\((l)\), y\_2\%\((l)\), \[CenterEllipsis], y\_\(N\_\(l - 1\)\)\%\((l)\))\), " ", \(l = 1, 2, \[CenterEllipsis], L - 1\), " ", " "} }], TraditionalForm]], "NumberedEquation", TextAlignment->Left, GridBoxOptions->{ColumnAlignments->{Left}}, CellTags->"Eq:mapmulti"], Cell[TextData[{ "where ", Cell[BoxData[ \(TraditionalForm\`l\)]], " denotes the layer of the hierarchy (numbering from 0 upwards), and ", Cell[BoxData[ \(TraditionalForm\`i\)]], " denotes the position of a node in the layer. Note that for notational \ convenience, after we transform from layer ", Cell[BoxData[ \(TraditionalForm\`l - 1\)]], " to layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], ", we gather together the ", Cell[BoxData[ \(TraditionalForm\`y\_i\%\((l)\)\)]], " to form a vector ", Cell[BoxData[ FormBox[ SuperscriptBox[ StyleBox["X", FontWeight->"Bold"], \((l)\)], TraditionalForm]]], " representing the state of the entire layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], " of the hierarchy." }], "Text"], Cell[TextData[{ "We then obtain the maximum entropy estimate ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " as" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], "=", "\[AlignmentMarker]", RowBox[{ RowBox[{"[", RowBox[{ FractionBox[ RowBox[{\(P\_1\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((0)\)], ")"}], \(\(p\_1\%\((1)\)\)(y\_1\%\((1)\))\)], FractionBox[ RowBox[{\(P\_2\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((0)\)], ")"}], \(\(p\_2\%\((1)\)\)(y\_2\%\((1)\))\)], "\[CenterEllipsis]", FractionBox[ RowBox[{\(P\_\(N\_0\)\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_0\), \((0)\)], ")"}], \(\(p\_\(N\_0\)\%\((1)\)\)( y\_\(N\_0\)\%\((1)\))\)]}], "]"}], "\n", "\[AlignmentMarker]", "[", RowBox[{ FractionBox[ RowBox[{\(P\_1\%\((1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((1)\)], ")"}], \(\(p\_1\%\((2)\)\)(y\_1\%\((2)\))\)], FractionBox[ RowBox[{\(P\_2\%\((1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((1)\)], ")"}], \(\(p\_2\%\((2)\)\)(y\_2\%\((2)\))\)], "\[CenterEllipsis]", FractionBox[ RowBox[{\(P\_\(N\_1\)\%\((1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_1\), \((1)\)], ")"}], \(\(p\_\(N\_1\)\%\((2)\)\)(y\_\(N\_1\)\%\((2)\))\)]}], "]"}]}], TraditionalForm], "\n", FormBox[ RowBox[{"\[AlignmentMarker]", RowBox[{"\[VerticalEllipsis]", "\n", "\[AlignmentMarker]", "[", RowBox[{ FractionBox[ RowBox[{\(P\_1\%\((L - 1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((L - 1)\)], ")"}], \(\(p\_1\%\((L)\)\)(y\_1\%\((L)\))\)], FractionBox[ RowBox[{\(P\_2\%\((L - 1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((L - 1)\)], ")"}], \(\(p\_2\%\((L)\)\)(y\_2\%\((L)\))\)], "\[CenterEllipsis]", FractionBox[ RowBox[{\(P\_\(N\_\(L - 1\)\)\%\((L - 1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_\(L - 1\)\), \((L - 1)\)], ")"}], \(\(p\_\(N\_\(L - 1\)\)\%\((L)\)\)( y\_\(N\_\(L - 1\)\)\%\((L)\))\)]}], "]"}]}], TraditionalForm], "\n", FormBox[ RowBox[{ "\[AlignmentMarker]", \(\(p\_\(12 \[CenterEllipsis]\ N\_\(L - \ 1\)\)\%\((L)\)\) \((y\_1\%\((L)\), y\_2\%\((L)\), \[CenterEllipsis], y\_\(N\_\(L - 1\)\)\%\((L)\))\)\)}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->"Eq:maxenthierarchy"], Cell[TextData[{ "We write ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " in a form that arranges the various factors differently from ", ButtonBox["equation", ButtonData:>"Eq:entropystat2bis", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:entropystat2bis"], ") and ", ButtonBox["equation", ButtonData:>"Eq:entropystatn", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:entropystatn"], "). We do this to reveal that ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " is simply the joint PDF ", Cell[BoxData[ \(TraditionalForm\`\(p\_\(12 \[CenterEllipsis]\ N\_\(L - \ 1\)\)\%\((L)\)\)(y\_1\%\((L)\), y\_2\%\((L)\), \[CenterEllipsis], y\_\(N\_\(L - 1\)\)\%\((L)\))\)]], " of the nodes in the final layer of the hierarchy, weighted by a product \ of factors which compensate for the compressions introduced by the various \ transformations ", Cell[BoxData[ FormBox[ RowBox[{\(y\_i\%\((l)\)\), "=", RowBox[{\(y\_i\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l - 1)\)], ")"}]}], TraditionalForm]]], "." }], "Text"], Cell[TextData[{ "Note that for ", Cell[BoxData[ \(TraditionalForm\`L = 1\)]], " ", ButtonBox["equation", ButtonData:>"Eq:maxenthierarchy", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:maxenthierarchy"], ") reduces to ", ButtonBox["equation", ButtonData:>"Eq:entropystatn", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:entropystatn"], "), where ", Cell[BoxData[ \(TraditionalForm\`N = N\_0\)]], ", ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold"], "=", SuperscriptBox[ StyleBox["X", FontWeight->"Bold"], \((0)\)]}], TraditionalForm]]], ", ", Cell[BoxData[ FormBox[ RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((0)\)], "=", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"]}], TraditionalForm]]], ", ", Cell[BoxData[ \(TraditionalForm\`y\_i\%\((1)\) = y\_i\)]], ", ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(P\_i\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((0)\)], ")"}], "=", RowBox[{\(P\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}]}], TraditionalForm]]], " and ", Cell[BoxData[ \(TraditionalForm\`\(p\_i\%\((1)\)\)(y\_i\%\((1)\)) = \(p\_i\)( y\_i)\)]], "." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". QUANTIFYING THE PERFORMANCE OF THE GIBBS DISTRIBUTION" }], "Section", CellTags->"Sect:3"], Cell[TextData[{ "In this section we define a measure of the quality of the maximum entropy \ estimate ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " given in ", ButtonBox["equation", ButtonData:>"Eq:maxenthierarchy", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:maxenthierarchy"], ") and we relate the properties of this measure to the properties of mutual \ information as described in the appendix in ", ButtonBox["section", ButtonData:>"Sect:6.2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:6.2"], ".", CounterBox["Subsection", "Sect:6.2"], "." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Relative entropy" }], "Subsection"], Cell["\<\ A measure of the quality of the maximum entropy estimate is relative entropy \ defined as\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{"G", "\[Congruent]", RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{"d", StyleBox["X", FontWeight->"Bold"]}]], " ", RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], RowBox[{"log", " ", "[", FractionBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}]], "]"}]}]}]}], "\[LessEqual]", "0"}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:relativeentropy"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " is given in ", ButtonBox["equation", ButtonData:>"Eq:maxenthierarchy", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:maxenthierarchy"], "). ", Cell[BoxData[ \(TraditionalForm\`G\)]], " has the following interpretation in terms of generalised Bernoulli \ trials. Define a model PDF ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], ", and use it as a source of independent samples of ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold"], TraditionalForm]]], ". Now construct a set containing a large number ", Cell[BoxData[ \(TraditionalForm\`M\)]], " of samples drawn from ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], ", and ask what is the probability that this set could have been drawn from \ ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " (irrespective of the order in which the individual samples were drawn). \ If we use ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " to denote the relative frequency of the various ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold"], TraditionalForm]]], " that is observed in this set of ", Cell[BoxData[ \(TraditionalForm\`M\)]], " samples, then a brief derivation (making use of Stirling's approximation \ ", Cell[BoxData[ \(TraditionalForm\`log\ \(M!\) \[TildeEqual] M\ Log\ M - M\)]], ") yields an answer ", Cell[BoxData[ \(TraditionalForm\`e\^\(M\ G\)\)]], " for the required probability. Thus the smaller ", Cell[BoxData[ \(TraditionalForm\`e\^\(M\ G\)\)]], " is (or, equivalently, the more negative ", Cell[BoxData[ \(TraditionalForm\`G\)]], " is) the more surprised we are that the observed ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], " is genuinely a possible relative frequency (ignoring ordering) of \ independent samples from a model ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", StyleBox["mem", FontSlant->"Italic"]], "(", StyleBox["X", FontWeight->"Bold"], ")"}], TraditionalForm]]], "." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Mutual information" }], "Subsection"], Cell[TextData[{ "In the appendix in ", ButtonBox["section", ButtonData:>"Sect:6.2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:6.2"], ".", CounterBox["Subsection", "Sect:6.2"], ". we outline some of the basic properties of mutual information, the most \ important of which is" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{\(I[X\_1; X\_2; \[CenterEllipsis]\ ; X\_n]\), "=", RowBox[{ RowBox[{"I", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ";", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ";", "\[CenterEllipsis]", " ", ";", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"]}], "]"}], "+", \(\[Sum]\+\(i = 0\)\%\(N - 1\)I[X\_\(p\_i\); X\_\(p\_i + 1\); \[CenterEllipsis]\ ; X\_\(p\_\(i + 1\)\)]\)}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:mutinfopartition2bis"], Cell[TextData[{ "where we define the notation in ", ButtonBox["equation", ButtonData:>"Eq:mutinfopartition1", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfopartition1"], "). ", ButtonBox["Equation", ButtonData:>"Eq:mutinfopartition2bis", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfopartition2bis"], ") is very intuitive, because it states that the total mutual information \ between all components ", Cell[BoxData[ \(TraditionalForm\`X\_i\)]], " of ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold"], TraditionalForm]]], " may be divided into two parts. The first part is the mutual information \ between separate clusters of components ", Cell[BoxData[ FormBox[ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], TraditionalForm]]], " of ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold"], TraditionalForm]]], ". The second part is the mutual information between the components ", Cell[BoxData[ \(TraditionalForm\`\((X\_\(p\_i + 1\), X\_\(p\_i + 2\), \[CenterEllipsis]\ , X\_\(p\_\(i + 1\)\))\)\)]], " of a single cluster, summed over clusters. We call these the \ inter-cluster and the intra-cluster mutual information, respectively. For \ layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], " of the hierarchy ", ButtonBox["equation", ButtonData:>"Eq:mutinfopartition2bis", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfopartition2bis"], ") becomes" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l)\)], "=", RowBox[{ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], "+", RowBox[{\(\[Sum]\+\(i = 1\)\%\(N\_l\)\), SubsuperscriptBox["I", StyleBox[\(intra, i\), FontSlant->"Italic"], \((l)\)]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"eq_mutinfosplitbis"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Decomposition of relative entropy into a sum of mutual informations" }], "Subsection"], Cell[TextData[{ "We now insert ", ButtonBox["equation", ButtonData:>"Eq:maxenthierarchy", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:maxenthierarchy"], ") into ", ButtonBox["equation", ButtonData:>"Eq:relativeentropy", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:relativeentropy"], ") to obtain some explicit results for the relative entropy. For the case \ ", Cell[BoxData[ \(TraditionalForm\`L = 1\)]], " we obtain (using ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold"], "=", SuperscriptBox[ StyleBox["X", FontWeight->"Bold"], \((0)\)]}], TraditionalForm]]], ")" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"G", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{ RowBox[{"-", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{"d", StyleBox["X", FontWeight->"Bold"]}]], " ", RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], RowBox[{"log", " ", "[", FractionBox[ RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], RowBox[{ RowBox[{\(P\_1\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((0)\)], ")"}], RowBox[{\(P\_2\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((0)\)], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(P\_\(N\_0\)\%\((0)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_0\), \((0)\)], ")"}]}]], "]"}]}]}]}], "\n", "\[AlignmentMarker]", "+", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{"d", StyleBox["X", FontWeight->"Bold"]}]], " ", RowBox[{"P", "(", StyleBox["X", FontWeight->"Bold"], ")"}], \(log\ [\(\(p\_\(12 \[CenterEllipsis]\ \ N\_0\)\%\((1)\)\)(y\_1\%\((1)\), y\_2\%\((1)\), \[CenterEllipsis]\ , \ y\_\(N\_0\)\%\((1)\))\)\/\(\(\(p\_1\%\((1)\)\)(y\_1\%\((1)\))\) \ \(\(p\_2\%\((1)\)\)(y\_2\%\((1)\))\) \[CenterEllipsis]\ \ \(\(p\_\(N\_0\)\%\((1)\)\)(y\_\(N\_0\)\%\((1)\))\)\)]\)}]}]}], "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((1)\)], "-", SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((1)\)]}], "\[LessEqual]", "0"}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->"Eq:relent2layer"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{\(y\_i\%\((1)\)\), "=", RowBox[{\(y\_i\%\((1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((0)\)], ")"}]}], TraditionalForm]]], " is to be understood. Note that we have used ", ButtonBox["equation", ButtonData:>"Eq:Itotal", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:Itotal"], ") for ", Cell[BoxData[ \(TraditionalForm\`l = 1\)]], " and ", ButtonBox["equation", ButtonData:>"Eq:Iinter", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:Iinter"], ") for ", Cell[BoxData[ \(TraditionalForm\`l = 0\)]], " to express ", Cell[BoxData[ \(TraditionalForm\`G\)]], " in terms of mutual information. The final inequality in ", ButtonBox["equation", ButtonData:>"Eq:relent2layer", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:relent2layer"], ") follows from ", ButtonBox["equation", ButtonData:>"Eq:mutinfoloss", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfoloss"], ") when we set ", Cell[BoxData[ \(TraditionalForm\`l = 0\)]], "." }], "Text"], Cell[TextData[{ "We may generalise ", ButtonBox["equation", ButtonData:>"Eq:relent2layer", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:relent2layer"], ") to the case ", Cell[BoxData[ \(TraditionalForm\`L \[GreaterEqual] 1\)]] }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"G", "=", RowBox[{\(\[Sum]\+\(l = 1\)\%\(L - 1\)\), RowBox[{"(", RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l + 1)\)], "-", SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)]}], ")"}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:relentllayer"], Cell[TextData[{ "We see that the mutual information loss ", Cell[BoxData[ FormBox[ RowBox[{ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], "-", SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l + 1)\)]}], TraditionalForm]]], " in transforming from ", Cell[BoxData[ \(TraditionalForm\`l\)]], " to layer ", Cell[BoxData[ \(TraditionalForm\`l + 1\)]], " determines to what extent ", Cell[BoxData[ \(TraditionalForm\`G\)]], " (and hence the quality of the Gibbs distribution model) is affected by \ that layer of the hierarchy. Finally, we may express ", ButtonBox["equation", ButtonData:>"Eq:relentllayer", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:relentllayer"], ") in an alternative form by using ", ButtonBox["equation", ButtonData:>"Eq:mutinfosplit", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfosplit"], ") to yield" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"G", "=", "\[AlignmentMarker]\[AlignmentMarker]", RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((L)\)], "-", SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((0)\)], "+", RowBox[{\(\[Sum]\+\(l = 1\)\%\(L - 1\)\), RowBox[{\(\[Sum]\+\(i = 1\)\%\(N\_l\)\), SubsuperscriptBox["I", StyleBox[\(intra, i\), FontSlant->"Italic"], \((l)\)]}]}]}], "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{ UnderscriptBox["\[Sum]", StyleBox["cliques", FontSlant->"Italic"]], SubscriptBox["I", StyleBox["clique", FontSlant->"Italic"]]}], "+", "constant"}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->"Eq:relentllayeralt"], Cell["\<\ where a \"clique\" is a set of sibling nodes in the hierarchical network.\ \>", "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". IMAGE-LIKE REPRESENTATION OF RELATIVE ENTROPY" }], "Section", CellTags->"Sect:4"], Cell[TextData[{ "In this section we apply the result in ", ButtonBox["equation", ButtonData:>"Eq:relentllayeralt", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:relentllayeralt"], ") to a binary tree of transformations of image data ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold"], TraditionalForm]]], ". There are many ways of connecting a binary tree to the pixels of an \ image. However, we restrict our attention to a binary tree that processes the \ pixels in the order shown in figure" }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/spie91/fig1.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:con3"], Cell["Order of pixel processing for an image.", "Caption"], Cell[TextData[{ "We also show how to represent the various contributions to ", Cell[BoxData[ \(TraditionalForm\`G\)]], " as images, which may then be inspected visually to locate contributions \ of interest, and we demonstrate this procedure using a Brodatz texture \ image." }], "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Clique mutual information" }], "Subsection"], Cell[TextData[{ "For a binary tree, each clique has a state that we represent as a vector \ ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ",", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]]}], ")"}], TraditionalForm]]], ", and a mutual information ", Cell[BoxData[ FormBox[ SubscriptBox["I", StyleBox["clique", FontSlant->"Italic"]], TraditionalForm]]], " given by" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["I", StyleBox["clique", FontSlant->"Italic"]], "=", RowBox[{ RowBox[{"I", "[", RowBox[{ SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ";", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]]}], "]"}], "=", RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox["dx", FontSlant->"Italic"], "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], SubsuperscriptBox[ StyleBox["dx", FontSlant->"Italic"], "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], RowBox[{"P", "(", RowBox[{ SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ",", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]]}], ")"}], RowBox[{"log", " ", "[", FractionBox[ RowBox[{"P", "(", RowBox[{ SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ",", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]]}], ")"}], RowBox[{ RowBox[{"P", "(", SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ")"}], RowBox[{"P", "(", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ")"}]}]], "]"}]}]}]}]}], TraditionalForm]], "NumberedEquation", SpanMaxSize->Infinity], Cell[TextData[{ "Note that ", Cell[BoxData[ FormBox[ SubscriptBox["I", StyleBox["clique", FontSlant->"Italic"]], TraditionalForm]]], " depends on ", Cell[BoxData[ FormBox[ RowBox[{"P", "(", RowBox[{ SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ",", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]]}], ")"}], TraditionalForm]]], ", which, in turn, depends on the co-occurrence matrix of the elements of \ the vector ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ",", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]]}], ")"}], TraditionalForm]]], ". Ideally, in order to evaluate the integral ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox["dx", FontSlant->"Italic"], "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], SubsuperscriptBox[ StyleBox["dx", FontSlant->"Italic"], "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], RowBox[{"P", "(", RowBox[{ SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ",", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]]}], ")"}], \((\[CenterEllipsis])\)}]}], TraditionalForm]]], " we require a representative ensemble of data ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold"], TraditionalForm]]], ", which would yield an approximation" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["I", StyleBox["clique", FontSlant->"Italic"]], "\[TildeEqual]", RowBox[{"\[LeftAngleBracket]", RowBox[{"log", " ", "[", FractionBox[ RowBox[{"P", "(", RowBox[{ SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ",", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]]}], ")"}], RowBox[{ RowBox[{"P", "(", SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ")"}], RowBox[{"P", "(", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ")"}]}]], "]"}], " ", "\[RightAngleBracket]"}]}], TraditionalForm]], "NumberedEquation", SpanMaxSize->Infinity], Cell[TextData[{ "where ", Cell[BoxData[ \(TraditionalForm\`\[LeftAngleBracket]\[CenterEllipsis]\ \[RightAngleBracket]\)]], " represents an ensemble average. However, in practice we use a single \ snapshot of the data, which yields only a very crude estimate of ", Cell[BoxData[ FormBox[ SubscriptBox["I", StyleBox["clique", FontSlant->"Italic"]], TraditionalForm]]], "." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Relative entropy image" }], "Subsection"], Cell[TextData[{ "It proves to be very convenient (and visually intuitive) to combine the \ single snapshot estimates of the various ", Cell[BoxData[ FormBox[ SubscriptBox["I", StyleBox["clique", FontSlant->"Italic"]], TraditionalForm]]], " into an image-like representation. Each clique ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ SubsuperscriptBox["x", "1", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]], ",", SubsuperscriptBox["x", "2", RowBox[{"(", StyleBox["cl", FontSlant->"Italic"], ")"}]]}], ")"}], TraditionalForm]]], " has a state that derives from the patch of input data ", Cell[BoxData[ FormBox[ StyleBox["X", FontWeight->"Bold"], TraditionalForm]]], " to which it is connected via the binary tree of transformations (as shown \ in ", ButtonBox["figure", ButtonData:>"Fig:con3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:con3"], ". Therefore a convenient way of making use of each ", Cell[BoxData[ FormBox[ SubscriptBox["I", StyleBox["clique", FontSlant->"Italic"]], TraditionalForm]]], " estimate is to propagate its value back down to its corresponding leaf \ nodes. The sum of the ", Cell[BoxData[ FormBox[ SubscriptBox["I", StyleBox["clique", FontSlant->"Italic"]], TraditionalForm]]], " values in ", ButtonBox["equation", ButtonData:>"Eq:relentllayeralt", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:relentllayeralt"], ") will then build up into an image as the various contributions accumulate \ at their corresponding leaf nodes. This image will have a blocky appearance \ because its contributions derive from a binary tree that is connected to the \ data pixels as shown in ", ButtonBox["figure", ButtonData:>"Fig:con3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:con3"], ", but we avoid this problem by forming a further average over an ensemble \ of closely related relative entropy images that are derived from binary trees \ that are translated to all possible positions relative to the image data." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Brodatz texture\t" }], "Subsection"], Cell[TextData[{ "In ", ButtonBox["figure", ButtonData:>"Fig:tex1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:tex1"], " we show an image of a Brodatz texture." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/spie91/fig2.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:tex1"], Cell["Optical image of a Brodatz texture.", "Caption"], Cell[TextData[{ "This image is ", Cell[BoxData[ \(TraditionalForm\`256\[Times]256\)]], " pixels in size, and each pixel is an integer in the interval ", Cell[BoxData[ \(TraditionalForm\`\(\([\)\(0, 255\)\(]\)\)\)]], " (i.e. 1 byte per pixel). The image is slightly unevenly illuminated and \ has a fairly low contrast, but nevertheless its statistical properties are \ almost translation invariant." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/spie91/fig3.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->"Fig:tex1anomaly"], Cell["Relative entropy images of Brodatz texture.", "Caption"], Cell[TextData[{ "In ", ButtonBox["figure", ButtonData:>"Fig:tex1anomaly", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:tex1anomaly"], " we show a set of relative entropy images, which are all displayed so that \ large values map to black pixels and small values map to white pixels. Each \ image is ", Cell[BoxData[ \(TraditionalForm\`256\[Times]256\)]], " pixels in size, and is scaled so that its pixel vales fill the interval \ ", Cell[BoxData[ \(TraditionalForm\`\(\([\)\(0, 255\)\(]\)\)\)]], ". Note how the these images become smoother as we progress from ", ButtonBox["figure", ButtonData:>"Fig:tex1anomaly", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:tex1anomaly"], "a to ", ButtonBox["figure", ButtonData:>"Fig:tex1anomaly", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:tex1anomaly"], "f, due to the increasing amount of averaging that occurs over the \ different placements of the binary tree." }], "Text"], Cell[TextData[{ ButtonBox["Figure", ButtonData:>"Fig:tex1anomaly", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:tex1anomaly"], "e and especially ", ButtonBox["figure", ButtonData:>"Fig:tex1anomaly", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:tex1anomaly"], "f reveal a highly localised anomaly in the original image. ", ButtonBox["Figure", ButtonData:>"Fig:tex1anomaly", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:tex1anomaly"], "f corresponds to a length scale of ", Cell[BoxData[ \(TraditionalForm\`8\[Times]8\)]], " pixels, which is the approximate size of the fault that is about ", Cell[BoxData[ \(TraditionalForm\`1/4\)]], " of the way down and slightly to the left of centre of ", ButtonBox["figure", ButtonData:>"Fig:tex1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:tex1"], ". The fault does not show up clearly on the other figures in ", ButtonBox["figure", ButtonData:>"Fig:tex1anomaly", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:tex1anomaly"], ", because their characteristic length scales are either too short or too \ long to be sensitive to the fault." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". CONCLUSIONS" }], "Section"], Cell[TextData[{ "In this paper we have summarised the derivation of a useful type of Gibbs \ distribution by constraining various of its marginal PDFs. In particular, we \ have presented the expression for the Gibbs distribution that emerges when we \ first of all develop a hierarchical transformation (e.g. a binary tree) of \ the original data, and then demand that the Gibbs distribution be consistent \ with the observed marginal PDFs between sibling nodes of the hierarchy. This \ type of situation is typical of multi-resolution analyses of data. We have \ also derived the relative entropy of this Gibbs distribution (relative to the \ ensemble of data vectors whose PDF it purports to model) to obtain a useful \ decomposition of the relative entropy as a sum of all of the mutual \ informations within the sets of sibling nodes of the hierarchy that defined \ the Gibbs distribution in the first place. It is important to note that each \ of these mutual informations is measured ", StyleBox["laterally", FontSlant->"Italic"], " within a layer of the hierarchy, rather than ", StyleBox["vertically", FontSlant->"Italic"], " between layers of the hierarchy. We have found that the relative entropy \ is (minus) the sum of the mutual infomation losses as we pass through the \ hierarchy, which corresponds to the fact that the nodes of the hierarchy do \ not preserve all the information about correlations between the clusters of \ pixel values from which they derive. Finally, we have shown how it is \ possible to display the various terms in the relative entropy expansion as \ images." }], "Text"], Cell["\<\ This type of approach where we investigate statistical correlations in \ textured (or cluttered) images by developing hierarchical transformations, \ and then measuring various statistical properties, is not itself new. \ However, our Gibbs distribution and the decomposition of its relative entropy \ as a sum of mutual informations, which then may be used to build up a \ relative entropy image, is novel. This gives insight into the nature of this \ type of Gibbs distribution model which may be used to interpret other, more \ complicated, Gibbs distribution models.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". APPENDIX" }], "Section", CellTags->"Sect:6"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Maximum entropy derivations" }], "Subsection", CellTags->"Sect:6.1"], Cell["Define the entropy functional", "Text"], Cell[BoxData[ FormBox[ RowBox[{"H", "\[Congruent]", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], RowBox[{"log", " ", "[", RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], "]"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:entropy2"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], " denotes an estimate of ", Cell[BoxData[ FormBox[ RowBox[{\(P\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], ". Now define a cost function to impose the required constraints on the \ marginal PDFs" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"C", "\[Congruent]", "\[AlignmentMarker]", RowBox[{"H", "+", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{ RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], " ", "[", RowBox[{ RowBox[{\(\[Lambda]\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "+", RowBox[{\(\[Lambda]\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], "]"}]}]}], "\n", "\[AlignmentMarker]", "+", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox["dy", FontSlant->"Italic"], "1"], SubscriptBox[ StyleBox["dy", FontSlant->"Italic"], "2"], RowBox[{\(\[Mu](y\_1, y\_2)\), " ", "[", RowBox[{"\[Integral]", RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "1"], SubscriptBox[ StyleBox[ RowBox[{"d", StyleBox["x", FontWeight->"Bold"]}]], "2"], RowBox[{"\[Delta]", "(", RowBox[{\(y\_1\), "-", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}]}], ")"}], RowBox[{"\[Delta]", "(", RowBox[{\(y\_2\), "-", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}]}], "]"}]}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker], Cell[TextData[{ "where we use Lagrange multiplier functions ", Cell[BoxData[ FormBox[ RowBox[{\(\[Lambda]\_i\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "i"], ")"}], TraditionalForm]]], " and ", Cell[BoxData[ \(TraditionalForm\`\[Mu](y\_1, y\_2)\)]], " to introduce the required constraints on ", Cell[BoxData[ FormBox[ RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], TraditionalForm]]], ". Functionally differentiating ", Cell[BoxData[ \(TraditionalForm\`C\)]], " yields" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[\(\[Delta]\ C\), RowBox[{"\[Delta]", " ", RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}]], "=", RowBox[{\(-1\), "-", RowBox[{"log", " ", "[", RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}], "]"}], "+", RowBox[{\(\[Lambda]\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "+", RowBox[{\(\[Lambda]\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "+", RowBox[{"\[Mu]", "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:entropyderiv2"], Cell[TextData[{ "Set ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"\[Delta]", " ", \(C/\[Delta]\), " ", RowBox[{\(Q\_12\), "(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"]}], ")"}]}], "=", "0"}], TraditionalForm]]], " to obtain the maximum entropy solution ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Q", RowBox[{ StyleBox["mem", FontSlant->"Italic"], StyleBox[",", FontSlant->"Plain"], StyleBox["12", FontSlant->"Plain"]}]], "(", StyleBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["1", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"]]}], FontWeight->"Bold"], ")"}], TraditionalForm]]], " as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ RowBox[{ SubscriptBox["Q", RowBox[{ StyleBox["mem", FontSlant->"Italic"], StyleBox[",", FontSlant->"Plain"], StyleBox["12", FontSlant->"Plain"]}]], "(", StyleBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["1", FontWeight->"Plain"]], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], StyleBox["2", FontWeight->"Plain"]]}], FontWeight->"Bold"], ")"}], "\[Proportional]", "\[AlignmentMarker]", RowBox[{"exp", " ", "[", RowBox[{ RowBox[{\(\[Lambda]\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], "+", RowBox[{\(\[Lambda]\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], "+", RowBox[{"\[Mu]", "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}]}], "]"}]}], "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{ RowBox[{\(P\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], RowBox[{\(P\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], FractionBox[ RowBox[{\(p\_12\), "(", RowBox[{ RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ",", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}]}], ")"}], RowBox[{ RowBox[{\(p\_1\), "(", RowBox[{\(y\_1\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ")"}], ")"}], RowBox[{\(p\_2\), "(", RowBox[{\(y\_2\), "(", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ")"}], ")"}]}]]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker], Cell["\<\ The fact that this solution is in closed form (i.e. determined, rather than \ undetermined, Lagrange multipliers) is the most important property of this \ type of maximum entropy model.\ \>", "Text", CellTags->"Ed:Change2"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " Mutual information" }], "Subsection", CellTags->"Sect:6.2"], Cell[TextData[{ "The basic definition of the mutual information ", Cell[BoxData[ \(TraditionalForm\`I[X; Y]\)]], " between a pair of random variables ", Cell[BoxData[ \(TraditionalForm\`X\)]], " and ", Cell[BoxData[ \(TraditionalForm\`X\)]], " may be given in three equivalent forms" }], "Text"], Cell[BoxData[ \(TraditionalForm\`I[X; Y] = H[X] + H[Y] - H[X, Y]\)], "NumberedEquation",\ TextAlignment->Left, CellTags->"Eq:mutinfobasic3"], Cell[BoxData[ \(TraditionalForm\`I[X; Y] = H[X] - H[X \[VerticalSeparator] Y]\)], "NumberedEquation", TextAlignment->Left, CellTags->{"Ed:Change3a", "Eq:mutinfobasic3a"}], Cell[BoxData[ \(TraditionalForm\`I[X; Y] = H[Y] - H[Y \[VerticalSeparator] X]\)], "NumberedEquation", TextAlignment->Left, CellTags->{"Ed:Change3b", "Eq:mutinfobasic3b"}], Cell[TextData[{ "where ", Cell[BoxData[ \(TraditionalForm\`H[r]\)]], " denotes the entropy functional applied to the PDF of random variable ", Cell[BoxData[ \(TraditionalForm\`r\)]], ". ", Cell[BoxData[ \(TraditionalForm\`e\^H[r]\)]], " is the effective number of states which is available to ", Cell[BoxData[ \(TraditionalForm\`r\)]], ", which we define as ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", "r", "]"}], TraditionalForm]]], ". Thus ", Cell[BoxData[ \(TraditionalForm\`e\^\(H[X] + H[Y]\)\)]], " is the value of ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X, Y\), "]"}], TraditionalForm]]], " ignoring any mutual dependencies between ", Cell[BoxData[ \(TraditionalForm\`X\)]], " and ", Cell[BoxData[ \(TraditionalForm\`Y\)]], ", and ", Cell[BoxData[ \(TraditionalForm\`e\^H[X, Y]\)]], " is the same including dependencies. Thus ", Cell[BoxData[ \(TraditionalForm\`e\^I[X; Y]\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:mutinfobasic3", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfobasic3"], ") measures the reduction in ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X, Y\), "]"}], TraditionalForm]]], " which occurs when mutual dependencies between ", Cell[BoxData[ \(TraditionalForm\`X\)]], " and ", Cell[BoxData[ \(TraditionalForm\`Y\)]], " are accounted for. The two alternative definitions in ", ButtonBox["equation", ButtonData:>"Eq:mutinfobasic3a", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfobasic3a"], ") and ", ButtonBox["equation", ButtonData:>"Eq:mutinfobasic3b", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfobasic3b"], ") may be similarly interpreted, although ", Cell[BoxData[ \(TraditionalForm\`X\)]], " and ", Cell[BoxData[ \(TraditionalForm\`Y\)]], " now enter asymmetrically." }], "Text"], Cell[TextData[{ "The most important property of mutual information is that it is an ", StyleBox["objective", FontSlant->"Italic"], " measure of correlations, as required of a consistent measure of \ information. Therefore it treats correlations, anti-correlations (and \ anything else) equivalently provided that they all lead to the same reduction \ in ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X, Y\), "]"}], TraditionalForm]]], ". Distinctions between various types of correlations made on the basic of \ measures other than ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X, Y\), "]"}], TraditionalForm]]], " are necessarily ", StyleBox["subjective", FontSlant->"Italic"], "." }], "Text"], Cell[TextData[{ "We may generalise ", ButtonBox["equation", ButtonData:>"Eq:mutinfobasic3", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfobasic3"], ") as follows" }], "Text"], Cell[BoxData[ \(TraditionalForm\`I[X\_1; X\_2; \[CenterEllipsis]\ ; X\_n] = \[Sum]\+\(i = 0\)\%n H[X\_i] - H[X\_1, X\_2, \[CenterEllipsis]\ , X\_n]\)], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:mutinfobasicn"], Cell[TextData[{ Cell[BoxData[ \(TraditionalForm\`e\^I[X\_1; X\_2; \[CenterEllipsis]\ ; X\_n]\)]], " measures the reduction in ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox["Z", StyleBox["eff", FontSlant->"Italic"]], "[", \(X\_1, X\_2, \[CenterEllipsis]\ , X\_n\), "]"}], TraditionalForm]]], " when mutual dependencies amongst the ", Cell[BoxData[ \(TraditionalForm\`X\_i\)]], " are taken into account." }], "Text"], Cell["\<\ We may further generalise these results by partitioning the random variables \ into mutually exclusive sets\ \>", "Text"], Cell[BoxData[{ FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold"], "=", "\[AlignmentMarker]", \((X\_1, X\_2, \[CenterEllipsis]\ , X\_n)\)}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], "=", "\[AlignmentMarker]", \((X\_1, X\_2, \[CenterEllipsis]\ , X\_\(p\_1\))\)}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], "=", "\[AlignmentMarker]", \((X\_\(p\_1 + 1\), X\_\(p\_1 + 2\), \[CenterEllipsis]\ , X\_\(p\_2\))\)}], TraditionalForm], "\n", FormBox["\[VerticalEllipsis]", TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"], "=", "\[AlignmentMarker]", \((X\_\(p\_\(N - 1\) + 1\), X\_\(p\_\(N - 1\) + 2\), \[CenterEllipsis]\ , X\_n)\)}], TraditionalForm], "\n", FormBox[ RowBox[{ StyleBox["X", FontWeight->"Bold"], "=", "\[AlignmentMarker]", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", " ", ",", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"]}], ")"}]}], TraditionalForm], "\n", FormBox[\(p\_0 = \[AlignmentMarker]0\), TraditionalForm], "\n", FormBox[\(p\_N = \[AlignmentMarker]n\), TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:mutinfopartition1"], Cell[BoxData[ FormBox[ RowBox[{\(I[X\_1; X\_2; \[CenterEllipsis]\ ; X\_n]\), "=", RowBox[{ RowBox[{"I", "[", RowBox[{ SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "1"], ";", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "2"], ";", "\[CenterEllipsis]", " ", ";", SubscriptBox[ StyleBox["x", FontWeight->"Bold"], "N"]}], "]"}], "+", \(\[Sum]\+\(i = 0\)\%\(N - 1\)I[X\_\(p\_i + 1\); X\_\(p\_i + 2\); \[CenterEllipsis]\ ; X\_\(p\_\(i + 1\)\)]\)}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->{"Eq:mutinfopartition2", "Ed:Change5"}], Cell[TextData[{ "Note that we have defined dummy partition boundaries ", Cell[BoxData[ \(TraditionalForm\`p\_0\)]], " and ", Cell[BoxData[ \(TraditionalForm\`p\_N\)]], " for convenience." }], "Text"], Cell["It is convenient to define two further quantities.", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l)\)], "\[Congruent]", "\[AlignmentMarker]", \(I[y\_1\%\((l)\); y\_2\%\((l)\); \[CenterEllipsis]\ ; y\_\(N\_\(l - 1\)\)\%\((l)\)]\)}], "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox["dy", FontSlant->"Italic"], "1", \((l)\)], SubsuperscriptBox[ StyleBox["dy", FontSlant->"Italic"], "2", \((l)\)], "\[CenterEllipsis]", " ", SubsuperscriptBox[ StyleBox["dy", FontSlant-> "Italic"], \(N\_\(l - 1\)\), \((l)\)], \(\(p\_\(12 \[CenterEllipsis]\ N\_\(l - \ 1\)\)\%\((l)\)\)(y\_1\%\((l)\), y\_2\%\((l)\), \[CenterEllipsis]\ , y\_\(N\_\(l - 1\)\)\%\((l)\))\), \(log\ [\(\(p\_\(12 \ \[CenterEllipsis]\ N\_\(l - 1\)\)\%\((l)\)\)(y\_1\%\((l)\), y\_2\%\((l)\), \ \[CenterEllipsis]\ , y\_\(N\_\(l - 1\)\)\%\((l)\))\)\/\(\(\(p\_1\%\((l)\)\)(y\ \_1\%\((l)\))\) \(\(p\_2\%\((l)\)\)(y\_2\%\((l)\))\) \[CenterEllipsis]\ \(\(p\ \_\(N\_\(l - 1\)\)\%\((l)\)\)(y\_\(N\_\(l - 1\)\)\%\((l)\))\)\)]\)}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->{"Ed:Change4", "Eq:Itotal"}], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], "\[Congruent]", "\[AlignmentMarker]", RowBox[{"I", "[", RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "1", \((l)\)], ";", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "2", \((l)\)], ";", "\[CenterEllipsis]", " ", ";", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(N\_l\), \((l)\)]}], "]"}]}], "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "1", \((l)\)], SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "2", \((l)\)], "\[CenterEllipsis]", " ", SubsuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], \(N\_l\), \((l)\)], RowBox[{\(P\^l\), "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold"], \((l)\)], ")"}], RowBox[{"log", " ", "[", FractionBox[ RowBox[{\(P\^l\), "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold"], \((l)\)], ")"}], RowBox[{ RowBox[{\(P\_1\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "1", \((l)\)], ")"}], RowBox[{\(P\_2\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "2", \((l)\)], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(P\_\(N\_l\)\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], \(N\_l\), \((l)\)], ")"}]}]], "]"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->"Eq:Iinter"], Cell[TextData[{ Cell[BoxData[ FormBox[ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l)\)], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:Itotal", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:Itotal"], ") and ", Cell[BoxData[ FormBox[ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:Iinter", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:Iinter"], ") are the total mutual information between ", StyleBox["all", FontSlant->"Italic"], " components of layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], ", and the mutual information between ", StyleBox["clusters", FontSlant->"Italic"], " of components of layer ", Cell[BoxData[ \(TraditionalForm\`l\)]], " respectively, and they satisfy the inequality" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], "\[GreaterEqual]", SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l + 1)\)], "\[GreaterEqual]", "0"}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:mutinfoloss"], Cell[TextData[{ "This can be proved by applying Jensen's inequality to the convex ", Cell[BoxData[ \(TraditionalForm\`\[Intersection]\)]], " logarithm function. ", ButtonBox["Equation", ButtonData:>"Eq:mutinfoloss", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfoloss"], ") states that one cannot gain any mutual information by transforming the \ ", Cell[BoxData[ FormBox[ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "i", \((l)\)], TraditionalForm]]], " via the functions ", Cell[BoxData[ FormBox[ RowBox[{\(y\_i\%\((l + 1)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "i", \((l)\)], ")"}], TraditionalForm]]], "." }], "Text"], Cell[TextData[{ "We obtain the multilayer generalisation of ", ButtonBox["equation", ButtonData:>"Eq:mutinfopartition2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfopartition2"], ") by defining" }], "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{\(N\_\(-1\)\), "=", RowBox[{"dim", "(", SuperscriptBox[ StyleBox["X", FontWeight->"Bold"], \((0)\)], ")"}]}], " ", RowBox[{ SuperscriptBox[ StyleBox["X", FontWeight->"Bold"], \((0)\)], "=", \((y\_1\%\((0)\), y\_2\%\((0)\), \[CenterEllipsis]\ , y\_\(N\_\(-1\)\)\%\((0)\))\)}], " ", RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((0)\)], "=", \(I[y\_1\%\((0)\); y\_2\%\((0)\); \[CenterEllipsis]\ ; y\_\(N\_\(-1\)\)\%\((0)\)]\)}]} }], TraditionalForm]], "NumberedEquation", TextAlignment->Left], Cell["and", "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l)\)], "=", \((x\_\(i\_1\)\%\((l)\), x\_\(i\_2\)\%\((l)\), \[CenterEllipsis]\ , x\_\(i\_\(n\_\(l\_i\)\)\)\%\((l)\))\)}], " ", \(l = 0, 1, \[CenterEllipsis]\ , L - 1\), " ", " ", \(i = 1, 2, \[CenterEllipsis]\ , N\_l\)} }], TraditionalForm]], "NumberedEquation", TextAlignment->Left], Cell[TextData[{ "Also define the mutual information between the components ", StyleBox["within a cluster", FontSlant->"Italic"], " as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SubsuperscriptBox["I", StyleBox[\(intra, i\), FontSlant->"Italic"], \((l)\)], "\[Congruent]", "\[AlignmentMarker]", \(I[x\_\(i\_1\)\%\((l)\); x\_\(i\_2\)\%\((l)\); \[CenterEllipsis]\ ; x\_\(i\_\(n\_\(l\_i\)\)\)\%\((l)\)]\)}], "\[IndentingNewLine]", "=", "\[AlignmentMarker]", RowBox[{"\[Integral]", RowBox[{ SubsuperscriptBox[ StyleBox["dx", FontSlant->"Italic"], \(i\_1\), \((l)\)], SubsuperscriptBox[ StyleBox["dx", FontSlant->"Italic"], \(i\_2\), \((l)\)], "\[CenterEllipsis]", " ", SubsuperscriptBox[ StyleBox["dx", FontSlant->"Italic"], \(i\_\(n\_\(l\_i\)\)\), \((l)\)], RowBox[{\(P\_i\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l)\)], ")"}], RowBox[{"log", " ", "[", FractionBox[ RowBox[{\(P\_i\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], "i", \((l)\)], ")"}], RowBox[{ RowBox[{\(P\_\(i\_1\)\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(i\_1\), \((l)\)], ")"}], RowBox[{\(P\_\(i\_2\)\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(i\_2\), \((l)\)], ")"}], "\[CenterEllipsis]", " ", RowBox[{\(P\_\(i\_\(n\_\(l\_i\)\)\)\%\((l)\)\), "(", SubsuperscriptBox[ StyleBox["x", FontWeight->"Bold"], \(i\_\(n\_\(l\_i\)\)\), \((l)\)], ")"}]}]], "]"}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->"Eq:Iintra"], Cell[TextData[{ "Finally we obtain the generalisation of ", ButtonBox["equation", ButtonData:>"Eq:mutinfopartition2", ButtonStyle->"Hyperlink"], " (", CounterBox["NumberedEquation", "Eq:mutinfopartition2"], ") as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ SubsuperscriptBox["I", StyleBox["total", FontSlant->"Italic"], \((l)\)], "=", RowBox[{ SubsuperscriptBox["I", StyleBox["inter", FontSlant->"Italic"], \((l)\)], "+", RowBox[{\(\[Sum]\+\(i = 1\)\%\(N\_l\)\), SubsuperscriptBox["I", StyleBox[\(intra, i\), FontSlant->"Italic"], \((l)\)]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Eq:mutinfosplit"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ". REFERENCES" }], "Section"], Cell[TextData[{ "1. H. Jeffreys, ", StyleBox["Theory of Probability", FontSlant->"Italic"], ", Clarendon Press, 1939." }], "Reference", CellTags->"Ref:Jeffreys1939"], Cell[TextData[{ "2. R. T. Cox, \"Probability, Frequency and Reasonable Expectation\", ", StyleBox["American Journal of Physics", FontSlant->"Italic"], ", vol. 14, pp. 1-13, 1946." }], "Reference", CellTags->"Ref:Cox1946"], Cell[TextData[{ "3. D. H. Ackley, G. E. Hinton and T. J. Sejnowski, \"A Learning Algorithm \ for Boltzmann Machines\", ", StyleBox["Cognitive Science", FontSlant->"Italic"], ", vol. 9, pp. 147-169, 1985." }], "Reference", CellTags->"Ref:AckleyHintonSejnowski1985"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/3815/3815.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "4. S. P. Luttrell, \"The Implications of Boltzmann-type Machines for SAR \ Data Processing: A Preliminary Survey\", ", StyleBox["RSRE Technical Memorandum No. 3815", FontSlant->"Italic"], ", 1985." }], "Reference", CellTags->"Ref:Luttrell1985"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/maxent88/maxent88.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "5. S. P. Luttrell, \"The Use of Bayesian and Entropic Methods in Neural \ Network Theory\", in \"Maximum Entropy and Bayesian Methods\", ed. J. \ Skilling, Kluwer, pp. 363-370, 1989." }], "Reference", CellTags->"Ref:Luttrell1989"], Cell[TextData[{ "6. S. P. Luttrell, \"Adaptive Cluster Expansion (ACE): An Adaptive \ Bayesian Network\", submitted to ", StyleBox["IEEE Transactions on Pattern Analysis and Machine Intelligence", FontSlant->"Italic"], "." }], "Reference", CellTags->"Ref:Luttrell1991"], Cell[TextData[{ "7. P. Brodatz, ", StyleBox["Textures - a Photographic Album for Artists and Designers", FontSlant->"Italic"], ", Dover, 1966." }], "Reference", CellTags->"Ref:Brodatz1966"] }, Closed]] }, Open ]] }, FrontEndVersion->"5.0 for Microsoft Windows", ScreenRectangle->{{0, 1280}, {0, 941}}, WindowSize->{665.375, 641}, WindowMargins->{{307.25, Automatic}, {Automatic, 50}}, Magnification->1, StyleDefinitions -> "Report.nb" ] (******************************************************************* Cached data follows. If you edit this Notebook file directly, not using Mathematica, you must remove the line containing CacheID at the top of the file. The cache data will then be recreated when you save this file from within Mathematica. *******************************************************************) (*CellTagsOutline CellTagsIndex->{ "Sect:2"->{ Cell[13252, 434, 108, 5, 41, "Section", CellTags->"Sect:2"]}, "Sect:2.1"->{ Cell[15085, 496, 141, 8, 70, "Subsection", CellTags->"Sect:2.1"]}, "Eq:margpdf2"->{ Cell[16362, 546, 1592, 48, 70, "NumberedEquation", CellTags->"Eq:margpdf2"]}, "Eq:map2"->{ Cell[18176, 606, 537, 15, 70, "NumberedEquation", CellTags->"Eq:map2"]}, "Eq:pdfmap2"->{ Cell[19071, 640, 1340, 37, 70, "NumberedEquation", CellTags->"Eq:pdfmap2"]}, "Eq:margpdfmap2"->{ Cell[21193, 706, 705, 18, 70, "NumberedEquation", CellTags->"Eq:margpdfmap2"]}, "Eq:entropystat2bis"->{ Cell[23602, 787, 1694, 48, 70, "NumberedEquation", CellTags->"Eq:entropystat2bis"]}, "Ed:Change1"->{ Cell[25299, 837, 2599, 81, 70, "Text", CellTags->"Ed:Change1"]}, "Eq:partition"->{ Cell[28485, 954, 578, 18, 70, "NumberedEquation", CellTags->"Eq:partition"]}, "Eq:margpdfn"->{ Cell[30133, 1020, 1389, 42, 70, "NumberedEquation", CellTags->"Eq:margpdfn"]}, "Eq:mapn"->{ Cell[31525, 1064, 280, 9, 70, "NumberedEquation", CellTags->"Eq:mapn"]}, "Eq:pdfmapn"->{ Cell[31808, 1075, 1758, 47, 70, "NumberedEquation", CellTags->"Eq:pdfmapn"]}, "Eq:margpdfmapn"->{ Cell[33569, 1124, 939, 26, 70, "NumberedEquation", CellTags->"Eq:margpdfmapn"]}, "Eq:entropystatn"->{ Cell[35114, 1177, 2265, 60, 70, "NumberedEquation", CellTags->"Eq:entropystatn"]}, "Eq:mapmulti"->{ Cell[38254, 1281, 1560, 37, 70, "NumberedEquation", CellTags->"Eq:mapmulti"]}, "Eq:maxenthierarchy"->{ Cell[40965, 1362, 3616, 87, 70, "NumberedEquation", CellTags->"Eq:maxenthierarchy"]}, "Sect:3"->{ Cell[47851, 1569, 140, 5, 64, "Section", CellTags->"Sect:3"]}, "Eq:relativeentropy"->{ Cell[49249, 1630, 959, 26, 70, "NumberedEquation", CellTags->"Eq:relativeentropy"]}, "Eq:mutinfopartition2bis"->{ Cell[53871, 1789, 800, 21, 70, "NumberedEquation", CellTags->"Eq:mutinfopartition2bis"]}, "eq_mutinfosplitbis"->{ Cell[56342, 1867, 566, 16, 70, "NumberedEquation", CellTags->"eq_mutinfosplitbis"]}, "Eq:relent2layer"->{ Cell[57896, 1928, 2674, 62, 70, "NumberedEquation", CellTags->"Eq:relent2layer"]}, "Eq:relentllayer"->{ Cell[62180, 2058, 514, 14, 70, "NumberedEquation", CellTags->"Eq:relentllayer"]}, "Eq:relentllayeralt"->{ Cell[63794, 2112, 1114, 28, 70, "NumberedEquation", CellTags->"Eq:relentllayeralt"]}, "Sect:4"->{ Cell[65057, 2150, 132, 5, 70, "Section", CellTags->"Sect:4"]}, "Fig:con3"->{ Cell[65781, 2175, 274, 10, 70, "NumberedFigure", CellTags->"Fig:con3"]}, "Fig:tex1"->{ Cell[76797, 2522, 274, 10, 70, "NumberedFigure", CellTags->"Fig:tex1"]}, "Fig:tex1anomaly"->{ Cell[77567, 2548, 281, 10, 70, "NumberedFigure", CellTags->"Fig:tex1anomaly"]}, "Sect:6"->{ Cell[82677, 2694, 95, 5, 70, "Section", CellTags->"Sect:6"]}, "Sect:6.1"->{ Cell[82797, 2703, 156, 8, 70, "Subsection", CellTags->"Sect:6.1"]}, "Eq:entropy2"->{ Cell[83004, 2715, 1189, 34, 70, "NumberedEquation", CellTags->"Eq:entropy2"]}, "Eq:entropyderiv2"->{ Cell[88956, 2885, 1575, 43, 70, "NumberedEquation", CellTags->"Eq:entropyderiv2"]}, "Ed:Change2"->{ Cell[94808, 3055, 235, 5, 70, "Text", CellTags->"Ed:Change2"]}, "Sect:6.2"->{ Cell[95080, 3065, 147, 8, 70, "Subsection", CellTags->"Sect:6.2"]}, "Eq:mutinfobasic3"->{ Cell[95560, 3088, 149, 4, 70, "NumberedEquation", CellTags->"Eq:mutinfobasic3"]}, "Ed:Change3a"->{ Cell[95712, 3094, 185, 4, 70, "NumberedEquation", CellTags->{"Ed:Change3a", "Eq:mutinfobasic3a"}]}, "Eq:mutinfobasic3a"->{ Cell[95712, 3094, 185, 4, 70, "NumberedEquation", CellTags->{"Ed:Change3a", "Eq:mutinfobasic3a"}]}, "Ed:Change3b"->{ Cell[95900, 3100, 185, 4, 70, "NumberedEquation", CellTags->{"Ed:Change3b", "Eq:mutinfobasic3b"}]}, "Eq:mutinfobasic3b"->{ Cell[95900, 3100, 185, 4, 70, "NumberedEquation", CellTags->{"Ed:Change3b", "Eq:mutinfobasic3b"}]}, "Eq:mutinfobasicn"->{ Cell[99610, 3234, 248, 5, 70, "NumberedEquation", CellTags->"Eq:mutinfobasicn"]}, "Eq:mutinfopartition1"->{ Cell[100500, 3263, 1820, 52, 70, "NumberedEquation", CellTags->"Eq:mutinfopartition1"]}, "Eq:mutinfopartition2"->{ Cell[102323, 3317, 817, 21, 70, "NumberedEquation", CellTags->{"Eq:mutinfopartition2", "Ed:Change5"}]}, "Ed:Change5"->{ Cell[102323, 3317, 817, 21, 70, "NumberedEquation", CellTags->{"Eq:mutinfopartition2", "Ed:Change5"}]}, "Ed:Change4"->{ Cell[103436, 3352, 1531, 35, 70, "NumberedEquation", CellTags->{"Ed:Change4", "Eq:Itotal"}]}, "Eq:Itotal"->{ Cell[103436, 3352, 1531, 35, 70, "NumberedEquation", CellTags->{"Ed:Change4", "Eq:Itotal"}]}, "Eq:Iinter"->{ Cell[104970, 3389, 3051, 78, 70, "NumberedEquation", CellTags->"Eq:Iinter"]}, "Eq:mutinfoloss"->{ Cell[109069, 3510, 398, 11, 70, "NumberedEquation", CellTags->"Eq:mutinfoloss"]}, "Eq:Iintra"->{ Cell[112201, 3611, 2182, 51, 70, "NumberedEquation", CellTags->"Eq:Iintra"]}, "Eq:mutinfosplit"->{ Cell[114635, 3675, 563, 16, 70, "NumberedEquation", CellTags->"Eq:mutinfosplit"]}, "Ref:Jeffreys1939"->{ Cell[115325, 3703, 177, 6, 70, "Reference", CellTags->"Ref:Jeffreys1939"]}, "Ref:Cox1946"->{ Cell[115505, 3711, 233, 6, 70, "Reference", CellTags->"Ref:Cox1946"]}, "Ref:AckleyHintonSejnowski1985"->{ Cell[115741, 3719, 276, 7, 70, "Reference", CellTags->"Ref:AckleyHintonSejnowski1985"]}, "Ref:Luttrell1985"->{ Cell[116020, 3728, 481, 16, 70, "Reference", CellTags->"Ref:Luttrell1985"]}, "Ref:Luttrell1989"->{ Cell[116504, 3746, 474, 15, 70, "Reference", CellTags->"Ref:Luttrell1989"]}, "Ref:Luttrell1991"->{ Cell[116981, 3763, 281, 7, 70, "Reference", CellTags->"Ref:Luttrell1991"]}, "Ref:Brodatz1966"->{ Cell[117265, 3772, 201, 6, 70, "Reference", CellTags->"Ref:Brodatz1966"]} } *) (*CellTagsIndex CellTagsIndex->{ {"Sect:2", 118145, 3799}, {"Sect:2.1", 118233, 3802}, {"Eq:margpdf2", 118329, 3805}, {"Eq:map2", 118432, 3808}, {"Eq:pdfmap2", 118533, 3811}, {"Eq:margpdfmap2", 118642, 3814}, {"Eq:entropystat2bis", 118758, 3817}, {"Ed:Change1", 118871, 3820}, {"Eq:partition", 118966, 3823}, {"Eq:margpdfn", 119073, 3826}, {"Eq:mapn", 119177, 3829}, {"Eq:pdfmapn", 119278, 3832}, {"Eq:margpdfmapn", 119388, 3835}, {"Eq:entropystatn", 119502, 3838}, {"Eq:mapmulti", 119614, 3841}, {"Eq:maxenthierarchy", 119729, 3844}, {"Sect:3", 119839, 3847}, {"Eq:relativeentropy", 119938, 3850}, {"Eq:mutinfopartition2bis", 120064, 3853}, {"eq_mutinfosplitbis", 120190, 3856}, {"Eq:relent2layer", 120308, 3859}, {"Eq:relentllayer", 120424, 3862}, {"Eq:relentllayeralt", 120542, 3865}, {"Sect:4", 120652, 3868}, {"Fig:con3", 120741, 3871}, {"Fig:tex1", 120840, 3874}, {"Fig:tex1anomaly", 120946, 3877}, {"Sect:6", 121050, 3880}, {"Sect:6.1", 121138, 3883}, {"Eq:entropy2", 121235, 3886}, {"Eq:entropyderiv2", 121348, 3889}, {"Ed:Change2", 121460, 3892}, {"Sect:6.2", 121550, 3895}, {"Eq:mutinfobasic3", 121652, 3898}, {"Ed:Change3a", 121763, 3901}, {"Eq:mutinfobasic3a", 121898, 3904}, {"Ed:Change3b", 122027, 3907}, {"Eq:mutinfobasic3b", 122162, 3910}, {"Eq:mutinfobasicn", 122296, 3913}, {"Eq:mutinfopartition1", 122416, 3916}, {"Eq:mutinfopartition2", 122543, 3919}, {"Ed:Change5", 122675, 3922}, {"Ed:Change4", 122807, 3925}, {"Eq:Itotal", 122928, 3928}, {"Eq:Iinter", 123049, 3931}, {"Eq:mutinfoloss", 123159, 3934}, {"Eq:Iintra", 123268, 3937}, {"Eq:mutinfosplit", 123379, 3940}, {"Ref:Jeffreys1939", 123496, 3943}, {"Ref:Cox1946", 123601, 3946}, {"Ref:AckleyHintonSejnowski1985", 123719, 3949}, {"Ref:Luttrell1985", 123842, 3952}, {"Ref:Luttrell1989", 123953, 3955}, {"Ref:Luttrell1991", 124064, 3958}, {"Ref:Brodatz1966", 124173, 3961} } *) (*NotebookFileOutline Notebook[{ Cell[CellGroupData[{ Cell[1776, 53, 26, 0, 70, "Section 1"], Cell[CellGroupData[{ Cell[1827, 57, 39, 0, 70, "Subsection"], Cell[1869, 59, 211, 8, 70, "Text"], Cell[2083, 69, 144, 3, 70, "Text"], Cell[2230, 74, 2043, 61, 70, "Text"], Cell[4276, 137, 1573, 48, 70, "Text"], Cell[5852, 187, 614, 23, 70, "Text"], Cell[6469, 212, 228, 8, 70, "Text"], Cell[6700, 222, 537, 18, 70, "Text"], Cell[7240, 242, 1247, 40, 70, "Text"], Cell[8490, 284, 494, 16, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[9033, 306, 73, 0, 120, "Title"], Cell[9109, 308, 192, 5, 114, "Author"], Cell[9304, 315, 134, 3, 29, "Text"], Cell[9441, 320, 923, 15, 214, "Abstract"], Cell[CellGroupData[{ Cell[10389, 339, 79, 4, 70, "Section 1"], Cell[10471, 345, 1461, 38, 70, "Text"], Cell[11935, 385, 1280, 44, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[13252, 434, 108, 5, 41, "Section", CellTags->"Sect:2"], Cell[13363, 441, 534, 15, 70, "Text"], Cell[13900, 458, 1160, 34, 70, "Text"], Cell[CellGroupData[{ Cell[15085, 496, 141, 8, 70, "Subsection", CellTags->"Sect:2.1"], Cell[15229, 506, 1130, 38, 70, "Text"], Cell[16362, 546, 1592, 48, 70, "NumberedEquation", CellTags->"Eq:margpdf2"], Cell[17957, 596, 216, 8, 70, "Text"], Cell[18176, 606, 537, 15, 70, "NumberedEquation", CellTags->"Eq:map2"], Cell[18716, 623, 352, 15, 70, "Text"], Cell[19071, 640, 1340, 37, 70, "NumberedEquation", CellTags->"Eq:pdfmap2"], Cell[20414, 679, 776, 25, 70, "Text"], Cell[21193, 706, 705, 18, 70, "NumberedEquation", CellTags->"Eq:margpdfmap2"], Cell[21901, 726, 1698, 59, 70, "Text"], Cell[23602, 787, 1694, 48, 70, "NumberedEquation", CellTags->"Eq:entropystat2bis"], Cell[25299, 837, 2599, 81, 70, "Text", CellTags->"Ed:Change1"] }, Closed]], Cell[CellGroupData[{ Cell[27935, 923, 225, 12, 70, "Subsection"], Cell[28163, 937, 319, 15, 70, "Text"], Cell[28485, 954, 578, 18, 70, "NumberedEquation", CellTags->"Eq:partition"], Cell[29066, 974, 1064, 44, 70, "Text"], Cell[30133, 1020, 1389, 42, 70, "NumberedEquation", CellTags->"Eq:margpdfn"], Cell[31525, 1064, 280, 9, 70, "NumberedEquation", CellTags->"Eq:mapn"], Cell[31808, 1075, 1758, 47, 70, "NumberedEquation", CellTags->"Eq:pdfmapn"], Cell[33569, 1124, 939, 26, 70, "NumberedEquation", CellTags->"Eq:margpdfmapn"], Cell[34511, 1152, 600, 23, 70, "Text"], Cell[35114, 1177, 2265, 60, 70, "NumberedEquation", CellTags->"Eq:entropystatn"], Cell[37382, 1239, 405, 16, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[37824, 1260, 126, 7, 70, "Subsection"], Cell[37953, 1269, 298, 10, 70, "Text"], Cell[38254, 1281, 1560, 37, 70, "NumberedEquation", CellTags->"Eq:mapmulti"], Cell[39817, 1320, 825, 27, 70, "Text"], Cell[40645, 1349, 317, 11, 70, "Text"], Cell[40965, 1362, 3616, 87, 70, "NumberedEquation", CellTags->"Eq:maxenthierarchy"], Cell[44584, 1451, 1606, 50, 70, "Text"], Cell[46193, 1503, 1609, 60, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[47851, 1569, 140, 5, 64, "Section", CellTags->"Sect:3"], Cell[47994, 1576, 990, 36, 70, "Text"], Cell[CellGroupData[{ Cell[49009, 1616, 121, 7, 70, "Subsection"], Cell[49133, 1625, 113, 3, 70, "Text"], Cell[49249, 1630, 959, 26, 70, "NumberedEquation", CellTags->"Eq:relativeentropy"], Cell[50211, 1658, 3161, 102, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[53409, 1765, 123, 7, 70, "Subsection"], Cell[53535, 1774, 333, 13, 70, "Text"], Cell[53871, 1789, 800, 21, 70, "NumberedEquation", CellTags->"Eq:mutinfopartition2bis"], Cell[54674, 1812, 1665, 53, 70, "Text"], Cell[56342, 1867, 566, 16, 70, "NumberedEquation", CellTags->"eq_mutinfosplitbis"] }, Closed]], Cell[CellGroupData[{ Cell[56945, 1888, 172, 7, 70, "Subsection"], Cell[57120, 1897, 773, 29, 70, "Text"], Cell[57896, 1928, 2674, 62, 70, "NumberedEquation", CellTags->"Eq:relent2layer"], Cell[60573, 1992, 1311, 51, 70, "Text"], Cell[61887, 2045, 290, 11, 70, "Text"], Cell[62180, 2058, 514, 14, 70, "NumberedEquation", CellTags->"Eq:relentllayer"], Cell[62697, 2074, 1094, 36, 70, "Text"], Cell[63794, 2112, 1114, 28, 70, "NumberedEquation", CellTags->"Eq:relentllayeralt"], Cell[64911, 2142, 97, 2, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[65057, 2150, 132, 5, 70, "Section", CellTags->"Sect:4"], Cell[65192, 2157, 586, 16, 70, "Text"], Cell[65781, 2175, 274, 10, 70, "NumberedFigure", CellTags->"Fig:con3"], Cell[66058, 2187, 58, 0, 70, "Caption"], Cell[66119, 2189, 299, 7, 70, "Text"], Cell[CellGroupData[{ Cell[66443, 2200, 130, 7, 70, "Subsection"], Cell[66576, 2209, 700, 22, 70, "Text"], Cell[67279, 2233, 2581, 65, 70, "NumberedEquation"], Cell[69863, 2300, 2321, 68, 70, "Text"], Cell[72187, 2370, 1239, 31, 70, "NumberedEquation"], Cell[73429, 2403, 435, 13, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[73901, 2421, 127, 7, 70, "Subsection"], Cell[74031, 2430, 2386, 67, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[76454, 2502, 122, 7, 70, "Subsection"], Cell[76579, 2511, 215, 9, 70, "Text"], Cell[76797, 2522, 274, 10, 70, "NumberedFigure", CellTags->"Fig:tex1"], Cell[77074, 2534, 54, 0, 70, "Caption"], Cell[77131, 2536, 433, 10, 70, "Text"], Cell[77567, 2548, 281, 10, 70, "NumberedFigure", CellTags->"Fig:tex1anomaly"], Cell[77851, 2560, 62, 0, 70, "Caption"], Cell[77916, 2562, 1060, 33, 70, "Text"], Cell[78979, 2597, 1308, 43, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[80336, 2646, 76, 4, 70, "Section"], Cell[80415, 2652, 1628, 26, 70, "Text"], Cell[82046, 2680, 594, 9, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[82677, 2694, 95, 5, 70, "Section", CellTags->"Sect:6"], Cell[CellGroupData[{ Cell[82797, 2703, 156, 8, 70, "Subsection", CellTags->"Sect:6.1"], Cell[82956, 2713, 45, 0, 70, "Text"], Cell[83004, 2715, 1189, 34, 70, "NumberedEquation", CellTags->"Eq:entropy2"], Cell[84196, 2751, 784, 25, 70, "Text"], Cell[84983, 2778, 3170, 77, 70, "NumberedEquation"], Cell[88156, 2857, 797, 26, 70, "Text"], Cell[88956, 2885, 1575, 43, 70, "NumberedEquation", CellTags->"Eq:entropyderiv2"], Cell[90534, 2930, 1287, 41, 70, "Text"], Cell[91824, 2973, 2981, 80, 70, "NumberedEquation"], Cell[94808, 3055, 235, 5, 70, "Text", CellTags->"Ed:Change2"] }, Closed]], Cell[CellGroupData[{ Cell[95080, 3065, 147, 8, 70, "Subsection", CellTags->"Sect:6.2"], Cell[95230, 3075, 327, 11, 70, "Text"], Cell[95560, 3088, 149, 4, 70, "NumberedEquation", CellTags->"Eq:mutinfobasic3"], Cell[95712, 3094, 185, 4, 70, "NumberedEquation", CellTags->{"Ed:Change3a", "Eq:mutinfobasic3a"}], Cell[95900, 3100, 185, 4, 70, "NumberedEquation", CellTags->{"Ed:Change3b", "Eq:mutinfobasic3b"}], Cell[96088, 3106, 2343, 85, 70, "Text"], Cell[98434, 3193, 946, 28, 70, "Text"], Cell[99383, 3223, 224, 9, 70, "Text"], Cell[99610, 3234, 248, 5, 70, "NumberedEquation", CellTags->"Eq:mutinfobasicn"], Cell[99861, 3241, 502, 15, 70, "Text"], Cell[100366, 3258, 131, 3, 70, "Text"], Cell[100500, 3263, 1820, 52, 70, "NumberedEquation", CellTags->"Eq:mutinfopartition1"], Cell[102323, 3317, 817, 21, 70, "NumberedEquation", CellTags->{"Eq:mutinfopartition2", "Ed:Change5"}], Cell[103143, 3340, 221, 8, 70, "Text"], Cell[103367, 3350, 66, 0, 70, "Text"], Cell[103436, 3352, 1531, 35, 70, "NumberedEquation", CellTags->{"Ed:Change4", "Eq:Itotal"}], Cell[104970, 3389, 3051, 78, 70, "NumberedEquation", CellTags->"Eq:Iinter"], Cell[108024, 3469, 1042, 39, 70, "Text"], Cell[109069, 3510, 398, 11, 70, "NumberedEquation", CellTags->"Eq:mutinfoloss"], Cell[109470, 3523, 877, 28, 70, "Text"], Cell[110350, 3553, 258, 9, 70, "Text"], Cell[110611, 3564, 855, 21, 70, "NumberedEquation"], Cell[111469, 3587, 19, 0, 70, "Text"], Cell[111491, 3589, 548, 13, 70, "NumberedEquation"], Cell[112042, 3604, 156, 5, 70, "Text"], Cell[112201, 3611, 2182, 51, 70, "NumberedEquation", CellTags->"Eq:Iintra"], Cell[114386, 3664, 246, 9, 70, "Text"], Cell[114635, 3675, 563, 16, 70, "NumberedEquation", CellTags->"Eq:mutinfosplit"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[115247, 3697, 75, 4, 70, "Section"], Cell[115325, 3703, 177, 6, 70, "Reference", CellTags->"Ref:Jeffreys1939"], Cell[115505, 3711, 233, 6, 70, "Reference", CellTags->"Ref:Cox1946"], Cell[115741, 3719, 276, 7, 70, "Reference", CellTags->"Ref:AckleyHintonSejnowski1985"], Cell[116020, 3728, 481, 16, 70, "Reference", CellTags->"Ref:Luttrell1985"], Cell[116504, 3746, 474, 15, 70, "Reference", CellTags->"Ref:Luttrell1989"], Cell[116981, 3763, 281, 7, 70, "Reference", CellTags->"Ref:Luttrell1991"], Cell[117265, 3772, 201, 6, 70, "Reference", CellTags->"Ref:Brodatz1966"] }, Closed]] }, Open ]] } ] *) (******************************************************************* End of Mathematica Notebook file. *******************************************************************)