(************** Content-type: application/mathematica ************** CreatedBy='Mathematica 5.0' Mathematica-Compatible Notebook This notebook can be used with any Mathematica-compatible application, such as Mathematica, MathReader or Publicon. The data for the notebook starts with the line containing stars above. To get the notebook into a Mathematica-compatible application, do one of the following: * Save the data starting with the line of stars above into a file with a name ending in .nb, then open the file inside the application; * Copy the data starting with the line of stars above to the clipboard, then use the Paste menu command inside the application. Data for notebooks contains only printable 7-bit ASCII and can be sent directly in email or through ftp in text mode. Newlines can be CR, LF or CRLF (Unix, Macintosh or MS-DOS style). NOTE: If you modify the data for this notebook not in a Mathematica- compatible application, you must delete the line below containing the word CacheID, otherwise Mathematica-compatible applications may try to use invalid cache data. For more information on notebooks and Mathematica-compatible applications, contact Wolfram Research: web: http://www.wolfram.com email: info@wolfram.com phone: +1-217-398-0700 (U.S.) Notebook reader applications are available free of charge from Wolfram Research. *******************************************************************) (*CacheID: 232*) (*NotebookFileLineBreakTest NotebookFileLineBreakTest*) (*NotebookOptionsPosition[ 117306, 3376]*) (*NotebookOutlinePosition[ 121918, 3502]*) (* CellTagsIndexPosition[ 120913, 3472]*) (*WindowFrame->Normal*) Notebook[{ Cell[CellGroupData[{ Cell["Notes", "Section 1"], Cell[CellGroupData[{ Cell["Editorial Changes", "Subsection"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change3", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change4", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "Equation reformatted to make it easier to read." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change1", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ \(TraditionalForm\`\(\(D\)\(\[Congruent]\)\)\)]], "\" prepended to equation" }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change2", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}], "/", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"]}], ")"}]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"]}], ")"}]}]], TraditionalForm]]], "\"" }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change5", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"1", "/", RowBox[{"(", RowBox[{"1", "+", RowBox[{"exp", "(", RowBox[{ RowBox[{"-", RowBox[{ RowBox[{ StyleBox["w", FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], ".", StyleBox["x", FontWeight->"Bold"]}]}], "-", RowBox[{"b", "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], ")"}]}], ")"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox["1", RowBox[{"1", "+", RowBox[{"exp", "(", RowBox[{ RowBox[{"-", RowBox[{ RowBox[{ StyleBox["w", FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], ".", StyleBox["x", FontWeight->"Bold"]}]}], "-", RowBox[{"b", "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], ")"}]}]], TraditionalForm]]], "\"" }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change6", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"1", "/", RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"]}], ")"}]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox["1", RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"]}], ")"}]}]], TraditionalForm]]], "\", and \"", Cell[BoxData[ FormBox[ RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], RowBox[{"(", RowBox[{"1", "/", RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"]}], ")"}]}]}], ")"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], FractionBox["1", RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"]}], ")"}]}]]}], TraditionalForm]]], "\"." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["A THEORY OF SELF-ORGANISING NEURAL NETWORKS", "Title"], Cell["\<\ S P Luttrell Defence Research Agency, Malvern, Worcs, WR14 3PS, UK. Email: luttrell@signal.dra.hmg.gb\ \>", "Author"], Cell["\<\ This appeared in Mathematics of Neural Networks: Models, Algorithms and \ Applications, Kluwer, Ellacott S W, Mason J C and Anderson I J (eds.), \ 240-244, 1997.\ \>", "Text"], Cell["\<\ \[Copyright] British Crown Copyright 1995/DRA. Published with the permission \ of the Controller of Her Britannic Majesty's Stationery Office.\ \>", "Text"], Cell[TextData[{ "The purpose of this paper is to present a probabilistic theory of \ self-organising networks based on the results published in [", ButtonBox["1", ButtonData:>"Ref:Luttrell1994a", ButtonStyle->"Hyperlink"], "]. This approach allows vector quantisers and topographic mappings to be \ treated as different limiting cases of the same theoretical framework. The \ full theoretical machinery allows a visual cortex-like network to be built." }], "Abstract"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Introduction" }], "Section 1"], Cell[TextData[{ "The purpose of this paper is to present a generalisation of the \ probabilistic approach to the static analysis of self-organising neural \ networks that appeared in [", ButtonBox["1", ButtonData:>"Ref:Luttrell1994a", ButtonStyle->"Hyperlink"], "]. In the simplest case the network has two layers: an input and an output \ layer. An input vector is used to clamp the pattern of activity of the nodes \ in the input layer, and the resulting pattern of individual \"firing\" events \ of the nodes in the output layer is described probabilistically. Finally, an \ attempt is made to reconstruct the pattern of activity in the input layer \ from knowledge of the location of the firing events in the output layer. This \ inversion from output to input is achieved by using Bayes' theorem to invert \ the probabilistic feed-forward mapping from input to output. A network \ objective function is then introduced in order to optimise the overall \ network performance. If the average Euclidean error between an input vector \ and its corresponding reconstruction is used as the objective function, then \ many standard self-organising networks emerge as special cases [", ButtonBox["1", ButtonData:>"Ref:Luttrell1994a", ButtonStyle->"Hyperlink"], ", ", ButtonBox["2", ButtonData:>"Ref:Luttrell1996", ButtonStyle->"Hyperlink"], "]." }], "Text"], Cell[TextData[{ "In ", ButtonBox["section", ButtonData:>"Sect:ObjectiveFunction", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:ObjectiveFunction"], " the network objective function is introduced, in ", ButtonBox["section", ButtonData:>"Sect:SimplifyObjectiveFunction", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:SimplifyObjectiveFunction"], " a simpler form is derived which is an upper bound to the true objective \ function, and in ", ButtonBox["section", ButtonData:>"Sect:DifferentiateObjectiveFunction", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:DifferentiateObjectiveFunction"], " the derivatives with respect to various parameters of this upper bound \ are derived. Finally, in ", ButtonBox["section", ButtonData:>"Sect:SpecialCases", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:SpecialCases"], " various standard neural networks are analysed within this framework." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " ", "Objective Function" }], "Section", CellTags->"Sect:ObjectiveFunction"], Cell[TextData[{ "The basic mathematical object is the objective function ", Cell[BoxData[ \(TraditionalForm\`D\)]], ", which is defined as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"D", "\[Congruent]", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "=", StyleBox["1", FontWeight->"Bold"]}]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", SuperscriptBox[ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], "\[Prime]"], RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{"Pr", "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}]}], ")"}], RowBox[{"Pr", "(", RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"]}], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->{"Eq:Objective", "Ed:Change1"}], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold"], TraditionalForm]]], " is the input vector and ", Cell[BoxData[ FormBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], TraditionalForm]]], " is its reconstruction, ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}], TraditionalForm]]], " are the locations in the output layer of ", Cell[BoxData[ \(TraditionalForm\`n\)]], " firing events, and ", Cell[BoxData[ FormBox[ SuperscriptBox[ RowBox[{"\[LeftBracketingBar]", RowBox[{"\[LeftBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"]}], "\[RightBracketingBar]"}], "\[RightBracketingBar]"}], "2"], TraditionalForm]]], " is the Euclidean distance between the input vector and its \ reconstruction. The various probabilities arise as follows: ", Cell[BoxData[ FormBox[ RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], \((\[CenterEllipsis])\)}]}], TraditionalForm]]], " integrates over the training set of input vectors, ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}]}], ")"}]}], TraditionalForm]]], " is the joint probability of ", Cell[BoxData[ \(TraditionalForm\`n\)]], " firing events at locations ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}], TraditionalForm]]], ", ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"]}], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}]}], TraditionalForm]]], " is the Bayes' inverse probability that input vector ", Cell[BoxData[ FormBox[ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], TraditionalForm]]], " is inferred as the cause of the ", Cell[BoxData[ \(TraditionalForm\`n\)]], " firing events, ", Cell[BoxData[ FormBox[ RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "=", "1"}]}], StyleBox["m", FontWeight->"Bold"]], \((\[CenterEllipsis])\)}], TraditionalForm]]], " sums over all possible locations (on an assumed rectangular lattice of \ size ", Cell[BoxData[ FormBox[ StyleBox["m", FontWeight->"Bold"], TraditionalForm]]], ") of the ", Cell[BoxData[ \(TraditionalForm\`n\)]], " firing events. The order in which the ", Cell[BoxData[ \(TraditionalForm\`n\)]], " firing events occurs will be assumed not to be observed, so that ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}]}], ")"}]}], TraditionalForm]]], " is a symmetric function of ", Cell[BoxData[ FormBox[ RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}], TraditionalForm]]], "." }], "Text"], Cell["\<\ A simplifying assumption will be made where the observed firing events are \ assumed to be statistically independent, so that\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"Pr", "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}]}], ")"}], "=", RowBox[{ RowBox[{"Pr", "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], RowBox[{"Pr", "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], RowBox[{"\[CenterEllipsis]Pr", "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:Independence"], Cell[TextData[{ "Normally, a simple form for ", Cell[BoxData[ FormBox[ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], TraditionalForm]]], " is used such as" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], "=", FractionBox[ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"]}], ")"}]}]]}], TraditionalForm]], "Text", CellTags->"Ed:Change2"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}], "\[GreaterEqual]", "0"}], TraditionalForm]]], ", which guarantees that the normalisation condition ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}]}], "=", "1"}], TraditionalForm]]], " holds. However, a more general expression will be used here based on the \ definition" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"Pr", "(", RowBox[{ RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ";", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"]}], ")"}], "\[Congruent]", FractionBox[ RowBox[{ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}], SubscriptBox["\[Delta]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]]}], RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"]}], ")"}]}]]}], TraditionalForm]], "NumberedEquation"], Cell[TextData[{ "which implies that ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}]], RowBox[{"Pr", "(", RowBox[{ RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ";", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"]}], ")"}]}], "=", "1"}], TraditionalForm]]], ", where ", Cell[BoxData[ FormBox[ RowBox[{"\[ScriptCapitalN]", RowBox[{"(", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " is the set of node locations that lie in a predefined \"neighbourhood\" \ of ", Cell[BoxData[ FormBox[ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], ". This neighbourhood can be used to introduce \"lateral inhibition\" \ between the firing neurons if the expression for ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " is written as a sum over ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ";", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"]}], ")"}]}], TraditionalForm]]], " as follows" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], "=", "\[AlignmentMarker]", RowBox[{\(1\/M\), RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], RowBox[{"Pr", "(", RowBox[{ RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ";", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"]}], ")"}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{"=", "\[AlignmentMarker]", RowBox[{\(1\/M\), RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}], RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], FractionBox["1", RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]], RowBox[{"Q", RowBox[{"(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"]}], ")"}]}]}]]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->{"Eq:LateralInhibition", "Ed:Change3"}], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " is the \"inverse neighbourhood\" of ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " defined as ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), "(", StyleBox["y", FontWeight->"Bold"], ")"}], "\[Congruent]", RowBox[{"{", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ":", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]}], "}"}]}], TraditionalForm]]], ", and ", Cell[BoxData[ \(TraditionalForm\`M\)]], " is the total number of locations in the output layer that have a non-zero \ neighbourhood size (", Cell[BoxData[ FormBox[ RowBox[{"M", "\[Congruent]", SubscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], ":", RowBox[{ RowBox[{"\[ScriptCapitalN]", "(", StyleBox["y", FontWeight->"Bold"], ")"}], "\[NotEqual]", "\[EmptySet]"}]}]]}], TraditionalForm]]], "). This expression for ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " is identical to one that arises in the context of optimising a special \ class of mixture distributions [", ButtonBox["3", ButtonData:>"Ref:Luttrell1994b", ButtonStyle->"Hyperlink"], "], and it satisfies ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}]}], "=", "1"}], TraditionalForm]]], ". The expresson for ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " in (", ButtonBox["4", ButtonData:>"Eq:LateralInhibition", ButtonStyle->"Hyperlink"], ") may readily be generalised to the case where the neighbourhood is \ non-uniformly weighted. The expression for ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in (", ButtonBox["1", ButtonData:>"Eq:Objective", ButtonStyle->"Hyperlink"], ") and the expression for ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " in (", ButtonBox["4", ButtonData:>"Eq:LateralInhibition", ButtonStyle->"Hyperlink"], ") are the two basic ingredients in the theory of self-organising neural \ networks presented in this paper." }], "Text"], Cell[TextData[{ "There is one further ingredient that proves to be very useful. ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " will be allowed to \"leak\" as follows [", ButtonBox["1", ButtonData:>"Ref:Luttrell1994a", ButtonStyle->"Hyperlink"], ", ", ButtonBox["3", ButtonData:>"Ref:Luttrell1994b", ButtonStyle->"Hyperlink"], "]" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], "\[LongRightArrow]", RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalL]\^\(-1\)\), "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}], RowBox[{"Pr", "(", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:Leakage"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " is the amount of probability that leaks from location ", Cell[BoxData[ FormBox[ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], " to location ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], ", and ", Cell[BoxData[ FormBox[ RowBox[{\(\[ScriptCapitalL]\^\(-1\)\), RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " is the \"inverse leakage neighbourhood\" of ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " defined as ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(\[ScriptCapitalL]\^\(-1\)\), "(", StyleBox["y", FontWeight->"Bold"], ")"}], "\[Congruent]", RowBox[{"{", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox[":", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[Element]", RowBox[{"\[ScriptCapitalL]", "(", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}]}], "}"}]}], TraditionalForm]]], ", where ", Cell[BoxData[ FormBox[ RowBox[{"\[ScriptCapitalL]", RowBox[{"(", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " is the \"leakage neighbourhood\" of ", Cell[BoxData[ FormBox[ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], TraditionalForm]]], ". The purpose of leakage is to allow the network output to be \"damaged\" \ in a controlled way, so that when the network is optimised it automatically \ becomes robust with respect to such damage. For instance, if each node is \ allowed to leak probability onto its neighbours in the output layer, then \ when the network is optimised the node properties become topographically \ ordered." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " ", "Simplify the Objective Function" }], "Section", CellTags->"Sect:SimplifyObjectiveFunction"], Cell[TextData[{ "The network objective function can be simplified to yield [", ButtonBox["1", ButtonData:>"Ref:Luttrell1994a", ButtonStyle->"Hyperlink"], "]" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "=", StyleBox["1", FontWeight->"Bold"]}]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{ RowBox[{"Pr", "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:ObjectiveSimplified"], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}]}], TraditionalForm]]], " is a \"reference vector\" defined as ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}], "\[Congruent]", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox[" ", FontWeight->"Bold"], RowBox[{"Pr", "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"]}], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}], " ", StyleBox["x", FontWeight->"Bold"]}]}]}], TraditionalForm]]], ". If ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}]}], TraditionalForm]]], " is treated as an independent variable (i.e. its definition is ignored) \ then minimisation of ", Cell[BoxData[ \(TraditionalForm\`D\)]], " with respect to it yields the result ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}]}], "=", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], StyleBox[" ", FontWeight->"Bold"], RowBox[{"Pr", "(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"]}], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}], " ", StyleBox["x", FontWeight->"Bold"]}]}]}], TraditionalForm]]], ", which is consistent with how it should have been defined anyway. This \ convenient trick allows the inverse probability ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[VerticalSeparator]", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"]}], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}]}], TraditionalForm]]], " to be eliminated henceforth, provided that ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in (", ButtonBox["6", ButtonData:>"Eq:ObjectiveSimplified", ButtonStyle->"Hyperlink"], ") is tacitly assumed to be minimised with respect to ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}]}], TraditionalForm]]], "." }], "Text"], Cell[TextData[{ Cell[BoxData[ \(TraditionalForm\`D\)]], " can be further simplified to the form ", Cell[BoxData[ \(TraditionalForm\`D = D\_1 + D\_2 - D\_3\)]], " [", ButtonBox["2", ButtonData:>"Ref:Luttrell1996", ButtonStyle->"Hyperlink"], "], where" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{\(D\_1\), "\[AlignmentMarker]", "\[Congruent]", RowBox[{\(2\/n\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{\(D\_2\), "\[AlignmentMarker]", "\[Congruent]", RowBox[{\(\(2 \((n - 1)\)\)\/n\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox[ SubscriptBox["y", StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold"], ",", RowBox[{ StyleBox[ SubscriptBox["y", StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox[ SubscriptBox["y", StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold"], ",", RowBox[{ StyleBox[ SubscriptBox["y", StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}]}], ")"}], RowBox[{ RowBox[{"(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", StyleBox[ SubscriptBox["y", StyleBox["1", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], ")"}], ".", RowBox[{"(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", StyleBox[ SubscriptBox["y", StyleBox["2", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}], ")"}]}]}]}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{\(D\_3\), "\[AlignmentMarker]", "\[Congruent]", RowBox[{"2", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "=", StyleBox["1", FontWeight->"Bold"]}]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{ RowBox[{"Pr", "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}], "-", RowBox[{\(1\/n\), RowBox[{\(\[Sum]\+\(i = 1\)\%n\), RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", StyleBox[ SubscriptBox["y", StyleBox["i", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}]}]}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker], Cell[TextData[{ "To obtain this result ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}]}], ")"}]}], TraditionalForm]]], " has been assumed to be symmetric under permutation of the locations ", Cell[BoxData[ FormBox[ RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], TraditionalForm]]], " (e.g. the locations, but not the order of occurrence of the ", Cell[BoxData[ \(TraditionalForm\`n\)]], " firing events is known). If the independence assumption in (", ButtonBox["2", ButtonData:>"Eq:Independence", ButtonStyle->"Hyperlink"], ") is now invoked, then ", Cell[BoxData[ \(TraditionalForm\`D\_2\)]], " may be simplified to" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{\(D\_2\), "=", RowBox[{\(\(2 \((n - 1)\)\)\/n\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:ObjectiveD2"], Cell[TextData[{ "The dependence of ", Cell[BoxData[ \(TraditionalForm\`D\_3\)]], " on the ", Cell[BoxData[ \(TraditionalForm\`n\)]], "-argument reference vectors ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], RowBox[{"(", RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "1"], ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "2"], ",", "\[CenterEllipsis]", ",", SubscriptBox[ StyleBox["y", FontWeight->"Bold"], "n"]}], ")"}]}], TraditionalForm]]], " is inconvenient, because the total number of such reference vectors is ", Cell[BoxData[ FormBox[ RowBox[{"\[ScriptCapitalO]", "(", SuperscriptBox[ RowBox[{"\[LeftBracketingBar]", StyleBox["m", FontWeight->"Bold"], "\[RightBracketingBar]"}], "n"], ")"}], TraditionalForm]]], ", where ", Cell[BoxData[ FormBox[ RowBox[{"\[LeftBracketingBar]", StyleBox["m", FontWeight->"Bold"], "\[RightBracketingBar]"}], TraditionalForm]]], " is the total number of output nodes (", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"\[LeftBracketingBar]", StyleBox["m", FontWeight->"Bold"], "\[RightBracketingBar]"}], "=", \(\(m\_1\) \(m\_2\) \[CenterEllipsis]\ m\_d\)}], TraditionalForm]]], " for a ", Cell[BoxData[ \(TraditionalForm\`d\)]], "-dimensional rectangular lattice of size ", Cell[BoxData[ FormBox[ StyleBox["m", FontWeight->"Bold"], TraditionalForm]]], "). However, the positivity of ", Cell[BoxData[ \(TraditionalForm\`D\_3\)]], ", together with ", Cell[BoxData[ \(TraditionalForm\`D = D\_1 + D\_2 - D\_3\)]], ", will be used to obtain an upper bound to ", Cell[BoxData[ \(TraditionalForm\`D\)]], " as ", Cell[BoxData[ \(TraditionalForm\`D \[LessEqual] D\_1 + D\_2\)]], ", which depends only on 1-argument reference vectors ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], ". The total number of 1-argument reference vectors is equal to ", Cell[BoxData[ FormBox[ RowBox[{"\[LeftBracketingBar]", StyleBox["m", FontWeight->"Bold"], "\[RightBracketingBar]"}], TraditionalForm]]], "." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Differentiate the Objective Function" }], "Section", CellTags->"Sect:DifferentiateObjectiveFunction"], Cell[TextData[{ "In order to implement an optimisation algorithm the derivatives of ", Cell[BoxData[ \(TraditionalForm\`D\_1\)]], " and ", Cell[BoxData[ \(TraditionalForm\`D\_2\)]], " with respect to the various parameters must be obtained. The expressions \ that are encountered when differentiating are rather cumbersome, but they \ have a simple structure which can be made clear by the introduction of the \ following notation" }], "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ SubscriptBox["L", RowBox[{ StyleBox["y", FontWeight->"Bold"], ",", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], "\[Congruent]", RowBox[{"Pr", "(", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["y", FontWeight->"Bold"]}], ")"}]}], RowBox[{ SubscriptBox["P", RowBox[{ StyleBox["y", FontWeight->"Bold"], StyleBox[",", FontWeight->"Plain"], StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], "\[Congruent]", RowBox[{"Pr", "(", RowBox[{ RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ";", StyleBox["y", FontWeight->"Bold"]}], ")"}]}]}, { RowBox[{ SubscriptBox["p", StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]}]], SubscriptBox["P", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ",", StyleBox["y", FontWeight->"Bold"]}]]}]}], RowBox[{ SubscriptBox[\((\(L\^T\) p)\), StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalL]\^\(-1\)\), RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]}]], RowBox[{ SubscriptBox["L", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ",", StyleBox["y", FontWeight->"Bold"]}]], SubscriptBox["p", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}]}, { RowBox[{ SubscriptBox[ StyleBox["d", FontWeight->"Bold"], StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]}], RowBox[{ SubscriptBox[ RowBox[{"(", RowBox[{"L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{"\[ScriptCapitalL]", "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], RowBox[{ SubscriptBox["L", RowBox[{ StyleBox["y", FontWeight->"Bold"], ",", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], SubscriptBox[ StyleBox["d", FontWeight->"Bold"], StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}]}, { RowBox[{ SubscriptBox[ RowBox[{"(", RowBox[{"P", " ", "L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], RowBox[{ SubscriptBox["P", RowBox[{ StyleBox["y", FontWeight->"Bold"], ",", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], " ", SubscriptBox[ RowBox[{"(", RowBox[{"L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}], RowBox[{ SubscriptBox[ RowBox[{"(", RowBox[{\(P\^T\), "P", " ", "L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]}]], RowBox[{ SubscriptBox["P", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ",", StyleBox["y", FontWeight->"Bold"]}]], " ", SubscriptBox[ RowBox[{"(", RowBox[{"P", " ", "L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}]}, { RowBox[{ SubscriptBox["e", StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", SuperscriptBox[ RowBox[{"\[LeftBracketingBar]", RowBox[{"\[LeftBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "\[RightBracketingBar]"}], "\[RightBracketingBar]"}], "2"]}], RowBox[{ SubscriptBox[\((L\ e)\), StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{"\[ScriptCapitalL]", "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], RowBox[{ SubscriptBox["L", RowBox[{ StyleBox["y", FontWeight->"Bold"], ",", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], SubscriptBox["e", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}]}, { RowBox[{ SubscriptBox[\((P\ L\ e)\), StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], RowBox[{ SubscriptBox["P", RowBox[{ StyleBox["y", FontWeight->"Bold"], ",", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], " ", SubscriptBox[\((L\ e)\), StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}], RowBox[{ SubscriptBox[\((\(P\^T\) P\ L\ e)\), StyleBox["y", FontWeight->"Bold"]], "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]}]], RowBox[{ SubscriptBox["P", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ",", StyleBox["y", FontWeight->"Bold"]}]], " ", SubscriptBox[\((P\ L\ e)\), StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}]}, { RowBox[{ StyleBox[ OverscriptBox["d", StyleBox["_", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Congruent]", RowBox[{ SubsuperscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{ SubscriptBox[\((\(L\^T\) p)\), StyleBox["y", FontWeight->"Bold"]], SubscriptBox[ StyleBox["d", FontWeight->"Bold"], StyleBox["y", FontWeight->"Bold"]]}]}]}], RowBox[{ RowBox[{"or", " ", StyleBox[ OverscriptBox["d", StyleBox["_", FontWeight->"Plain"]], FontWeight->"Bold"]}], "\[Congruent]", RowBox[{ SubsuperscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], SubscriptBox[ RowBox[{"(", RowBox[{"P", " ", "L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox["y", FontWeight->"Bold"]]}]}]} }], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, GridBoxOptions->{ColumnAlignments->{Left}}], Cell[TextData[{ "which allows (", ButtonBox["5", ButtonData:>"Eq:Leakage", ButtonStyle->"Hyperlink"], ") to be written as ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{ RowBox[{"(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], "\[LongRightArrow]", \(1\/M\)}], SubscriptBox[\((\(L\^T\) p)\), StyleBox["y", FontWeight->"Bold"]]}], TraditionalForm]]], ". ", Cell[BoxData[ \(TraditionalForm\`D\_1\)]], " and ", Cell[BoxData[ \(TraditionalForm\`D\_2\)]], " may be differentiated with respect to ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " to obtain" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[\(\[PartialD]D\_1\), RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], "\[AlignmentMarker]", "=", RowBox[{ RowBox[{\(-\(4\/\(n\ M\)\)\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], SubscriptBox[\((\(L\^T\) p)\), StyleBox["y", FontWeight->"Bold"]], SubscriptBox[ StyleBox["d", FontWeight->"Bold"], StyleBox["y", FontWeight->"Bold"]], "\n", FractionBox[\(\[PartialD]D\_2\), RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]]}]}]}], "\[AlignmentMarker]", "=", RowBox[{\(-\(\(4 \((n - 1)\)\)\/\(n\ M\^2\)\)\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], SubscriptBox[\((\(L\^T\) p)\), StyleBox["y", FontWeight->"Bold"]], StyleBox[ OverscriptBox["d", StyleBox["_", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:DerivativeXprime"], Cell[TextData[{ Cell[BoxData[ \(TraditionalForm\`D\_1\)]], " and ", Cell[BoxData[ \(TraditionalForm\`D\_2\)]], " may be functionally varied with respect to ", Cell[BoxData[ FormBox[ RowBox[{"log", " ", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " to obtain" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], "1"], "\[AlignmentMarker]", "=", RowBox[{\(2\/\(n\ M\)\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{"\[Delta]", " ", "log", " ", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}], RowBox[{"(", RowBox[{ RowBox[{ SubscriptBox["p", StyleBox["y", FontWeight->"Bold"]], " ", SubscriptBox[\((L\ e)\), StyleBox["y", FontWeight->"Bold"]]}], "-", SubscriptBox[\((\(P\^T\) P\ L\ e)\), StyleBox["y", FontWeight->"Bold"]]}], ")"}]}]}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ SubscriptBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], "2"], "\[AlignmentMarker]", "=", RowBox[{\(\(4 \((n - 1)\)\)\/\(n\ M\^2\)\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{"\[Delta]", " ", "log", " ", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}], RowBox[{ RowBox[{"(", RowBox[{ RowBox[{ SubscriptBox["p", StyleBox["y", FontWeight->"Bold"]], " ", SubscriptBox[ RowBox[{"(", RowBox[{"L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox["y", FontWeight->"Bold"]]}], "-", SubscriptBox[ RowBox[{"(", RowBox[{\(P\^T\), "P", " ", "L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox["y", FontWeight->"Bold"]]}], ")"}], ".", StyleBox[ OverscriptBox["d", StyleBox["_", FontWeight->"Plain"]], FontWeight->"Bold"]}]}]}]}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker], Cell[TextData[{ "If ", Cell[BoxData[ FormBox[ RowBox[{"Q", RowBox[{"(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " is assumed to be a sigmoid function" }], "Text"], Cell[TextData[Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}], "=", FractionBox["1", RowBox[{"1", "+", RowBox[{"exp", "(", RowBox[{ RowBox[{"-", RowBox[{ RowBox[{ StyleBox["w", FontWeight->"Bold"], "(", StyleBox["y", FontWeight->"Bold"], ")"}], ".", StyleBox["x", FontWeight->"Bold"]}]}], "-", RowBox[{"b", "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], ")"}]}]]}], TraditionalForm]]]], "Text", CellTags->"Ed:Change5"], Cell[TextData[{ "then the derivatives of ", Cell[BoxData[ \(TraditionalForm\`D\_1\)]], " and ", Cell[BoxData[ \(TraditionalForm\`D\_2\)]], " with respect to the \"weight\" vector ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["w", FontWeight->"Bold"], RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}], TraditionalForm]]], " and \"bias\" ", Cell[BoxData[ FormBox[ RowBox[{"b", "(", StyleBox["y", FontWeight->"Bold"], ")"}], TraditionalForm]]], " may be written as" }], "Text"], Cell[BoxData[{ FormBox[ RowBox[{ FractionBox[\(\[PartialD]D\_1\), RowBox[{"\[PartialD]", RowBox[{"(", "\[NegativeThinSpace]", GridBox[{ { RowBox[{"b", RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]}, { RowBox[{ StyleBox["w", FontWeight->"Bold"], RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]} }], "\[NegativeThinSpace]", ")"}]}]], "\[AlignmentMarker]", "=", RowBox[{\(2\/\(n\ M\)\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{ RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], " ", "[", RowBox[{ RowBox[{"(", RowBox[{ RowBox[{ SubscriptBox["p", StyleBox["y", FontWeight->"Bold"]], " ", SubscriptBox[\((L\ e)\), StyleBox["y", FontWeight->"Bold"]]}], "-", SubscriptBox[\((\(P\^T\) P\ L\ e)\), StyleBox["y", FontWeight->"Bold"]]}], ")"}], RowBox[{"(", RowBox[{"1", "-", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}]}], ")"}], RowBox[{"(", "\[NegativeThinSpace]", GridBox[{ {"1"}, { StyleBox["x", FontWeight->"Bold"]} }], "\[NegativeThinSpace]", ")"}]}], "]"}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ FractionBox[\(\[PartialD]D\_2\), RowBox[{"\[PartialD]", RowBox[{"(", "\[NegativeThinSpace]", GridBox[{ { RowBox[{"b", RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]}, { RowBox[{ StyleBox["w", FontWeight->"Bold"], RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]} }], "\[NegativeThinSpace]", ")"}]}]], "\[AlignmentMarker]", "=", RowBox[{\(\(4 \((n - 1)\)\)\/\(n\ M\^2\)\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{ RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], " ", "[", RowBox[{ RowBox[{ RowBox[{"(", RowBox[{ RowBox[{ SubscriptBox["p", StyleBox["y", FontWeight->"Bold"]], " ", SubscriptBox[ RowBox[{"(", RowBox[{"L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox["y", FontWeight->"Bold"]]}], "-", SubscriptBox[ RowBox[{"(", RowBox[{\(P\^T\), "P", " ", "L", " ", StyleBox["d", FontWeight->"Bold"]}], ")"}], StyleBox["y", FontWeight->"Bold"]]}], ")"}], ".", RowBox[{ StyleBox[ OverscriptBox["d", StyleBox["_", FontWeight->"Plain"]], FontWeight->"Bold"], "(", RowBox[{"1", "-", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["y", FontWeight->"Bold"]}], ")"}]}], ")"}]}], RowBox[{"(", "\[NegativeThinSpace]", GridBox[{ {"1"}, { StyleBox["x", FontWeight->"Bold"]} }], "\[NegativeThinSpace]", ")"}]}], "]"}]}]}]}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, SpanMaxSize->Infinity, CellTags->{"Eq:DerivativeWB", "Ed:Change4"}], Cell[TextData[{ "The gradients in (", ButtonBox["10", ButtonData:>"Eq:DerivativeXprime", ButtonStyle->"Hyperlink"], ") and (", ButtonBox["12", ButtonData:>"Eq:DerivativeWB", ButtonStyle->"Hyperlink"], ") may be used to implement a gradient descent algorithm for optimising ", Cell[BoxData[ \(TraditionalForm\`D\_1 + D\_2\)]], ", which then leads to a least upper bound on the full objective function \ ", Cell[BoxData[ \(TraditionalForm\`D\)]], " (", Cell[BoxData[ \(TraditionalForm\`\(\(=\)\(D\_1 + D\_2 - D\_3\)\)\)]], ")." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " ", "Special Cases" }], "Section", CellTags->"Sect:SpecialCases"], Cell["\<\ Various standard results that are special cases of the model presented above \ are discussed in the following subsections.\ \>", "Text"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", "Vector Quantiser and Topographic Mapping" }], "Subsection"], Cell[TextData[{ "Assume ", Cell[BoxData[ \(TraditionalForm\`n = 1\)]], " so that only 1 firing event is observed so that ", Cell[BoxData[ \(TraditionalForm\`D\_2 = \(D\_3 = 0\)\)]], ", ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[Element]", RowBox[{ RowBox[{"\[ScriptCapitalN]", "(", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ")"}], " ", RowBox[{"\[ForAll]", StyleBox["y", FontWeight->"Bold"]}]}]}], ",", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}], TraditionalForm]]], " so that the neighbourhood embraces all of the output nodes, and \ probability leakage of the type given in (", ButtonBox["5", ButtonData:>"Eq:Leakage", ButtonStyle->"Hyperlink"], ") is allowed. Then ", Cell[BoxData[ \(TraditionalForm\`D\)]], " reduces to ", Cell[BoxData[ FormBox[ RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold"], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftDoubleBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], "(", StyleBox["y", FontWeight->"Bold"], ")"}]}], "\[RightDoubleBracketingBar]"}], "2"]}]}]}]}]}]}], TraditionalForm]]], " [", ButtonBox["1", ButtonData:>"Ref:Luttrell1994a", ButtonStyle->"Hyperlink"], "], which leads to a behaviour that is very similar to the topographic \ mappings described in [", ButtonBox["5", ButtonData:>"Ref:Kohonen1984", ButtonStyle->"Hyperlink"], "], where ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " now corresponds to the topographic neighbourhood function. In the limit \ where ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}]}], ")"}], "=", SubscriptBox["\[Delta]", RowBox[{ StyleBox["y", FontWeight->"Bold"], ",", RowBox[{ StyleBox["y", FontWeight->"Bold"], "(", StyleBox["x", FontWeight->"Bold"], ")"}]}]]}], TraditionalForm]]], " this reduces to the criterion for optimising a vector quantiser [", ButtonBox["4", ButtonData:>"Ref:LindeBuzoGray1980", ButtonStyle->"Hyperlink"], "]." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], ".", CounterBox["Subsection"], " ", "Visual Cortex Network" }], "Subsection"], Cell[TextData[{ "A \"visual cortex'\"-like network can be built if the full theoretical \ machinery presented earlier is used. This network has many of the emergent \ properties of the mammalian visual cortex, such as orientation maps, \ centre-on/surround-off detectors, dominance stripes, etc (see e.g. [", ButtonBox["7", ButtonData:>"Ref:Goodhill1992", ButtonStyle->"Hyperlink"], "] for a review of these phenomena). There is an input layer with a pattern \ of activity representing the input vector, an output layer with nodes firing \ in response to feed-forward connections (i.e. a \"recognition\" model), and a \ mechanism for reconstructing the input from the firing events via feed-back \ connections (i.e. a \"generative\" model). The output layer has lateral \ inhibition implemented as in (", ButtonBox["4", ButtonData:>"Eq:LateralInhibition", ButtonStyle->"Hyperlink"], "); the neighbourhood of node ", Cell[BoxData[ FormBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], TraditionalForm]]], " has an associated inhibition factor ", Cell[BoxData[ FormBox[ FractionBox["1", RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"]}], ")"}]}]], TraditionalForm]]], ", and the overall inhibition factor for node ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " is ", Cell[BoxData[ FormBox[ RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", RowBox[{\(\[ScriptCapitalN]\^\(-1\)\), "(", StyleBox["y", FontWeight->"Bold"], ")"}]}]], FractionBox["1", RowBox[{ UnderscriptBox["\[Sum]", RowBox[{ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"], "\[Element]", RowBox[{"\[ScriptCapitalN]", "(", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], ")"}]}]], RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold"], "\[VerticalSeparator]", SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[DoublePrime]"]}], ")"}]}]]}], TraditionalForm]]], ", which is the sum of the inhibition factors over all nodes ", Cell[BoxData[ FormBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Bold"], "\[Prime]"], TraditionalForm]]], " that have node ", Cell[BoxData[ FormBox[ StyleBox["y", FontWeight->"Bold"], TraditionalForm]]], " in their neighbourhood. This scheme for introducing lateral inhibition is \ discussed in greater detail in [", ButtonBox["3", ButtonData:>"Ref:Luttrell1994b", ButtonStyle->"Hyperlink"], "]. The leakage introduced by ", Cell[BoxData[ FormBox[ RowBox[{"Pr", RowBox[{"(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox[ SuperscriptBox["y", StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}], ")"}]}], TraditionalForm]]], " induces topographical ordering as usual." }], "Text", CellTags->"Ed:Change6"], Cell[TextData[{ "In the limit ", Cell[BoxData[ \(TraditionalForm\`n\[LongRightArrow]1\)]], " where ", Cell[BoxData[ \(TraditionalForm\`D\_1\)]], " is dominant, this network behaves like a topographic mapping network, \ except that the output layer splits up into a number of \"domains\" each of \ which is typically a lateral inhibition length in size, and each of which \ forms a separate topographic mapping. These domains are seamlessly joined \ together, so no domain boundaries are actually visible. In the limit ", Cell[BoxData[ \(TraditionalForm\`n\[LongRightArrow]\[Infinity]\)]], " where ", Cell[BoxData[ \(TraditionalForm\`D\_2\)]], " is dominant, this network approximates its input as a superposition of \ reference vectors ", Cell[BoxData[ FormBox[ RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontWeight->"Bold"], "=", StyleBox["1", FontWeight->"Bold"]}], StyleBox["m", FontWeight->"Bold"]], RowBox[{ RowBox[{"Pr", "(", RowBox[{ StyleBox["y", FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ")"}], SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Italic"], "\[Prime]"], RowBox[{"(", StyleBox["y", FontWeight->"Bold"], ")"}]}]}], TraditionalForm]]], " (see (", ButtonBox["8", ButtonData:>"Eq:ObjectiveD2", ButtonStyle->"Hyperlink"], ")). Thus the network is capable of explaining the input in terms of \ multiple causes." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["Conclusions", "Section"], Cell["\<\ A single theoretical framework has been shown to describe a number of \ standard self-organising neural networks. This makes it easy to understand \ the relationship between these neural networks, and it provides a useful \ framework for analysing their properties.\ \>", "Text"] }, Open ]], Cell[CellGroupData[{ Cell["REFERENCES", "Section"], Cell[TextData[{ "\n", Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/bayessom.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "[1]\tLuttrell S P, ", StyleBox["A Bayesian analysis of self-organising maps", FontSlant->"Italic"], ", Neural Computation, Vol. 6(5) (1994), pp767-794.\t" }], "Reference", CellTags->"Ref:Luttrell1994a"], Cell[TextData[{ "[2]\tLuttrell S P, ", StyleBox["Designing analysable networks", FontSlant->"Italic"], ", Handbook of Neural Computation, OUP (1996).\t" }], "Reference", CellTags->"Ref:Luttrell1996"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/pmd/pmd.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "[3]\tLuttrell S P, ", StyleBox["Partitioned mixture distribution: an adaptive Bayesian network \ for low-level image processing", FontSlant->"Italic"], ", IEE Proc. Vision Image Signal Processing, Vol. 141(4) (1994), \ pp251-260." }], "Reference", CellTags->"Ref:Luttrell1994b"], Cell[TextData[{ "[4]\tLinde Y, Buzo A and Gray R M, ", StyleBox["An algorithm for vector quantiser design", FontSlant->"Italic"], ", IEEE Trans. COM, Vol. 28(1) (1980), pp84-95.\t" }], "Reference", CellTags->"Ref:LindeBuzoGray1980"], Cell[TextData[{ "[5]\tKohonen T, ", StyleBox["Self organisation and associative memory", FontSlant->"Italic"], ", Springer-Verlag (1984).\t" }], "Reference", CellTags->"Ref:Kohonen1984"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/tvq/tvq.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "[6]\tLuttrell S P, ", StyleBox["Derivation of a class of training algorithms", FontSlant->"Italic"], ", IEEE Trans. NN, Vol. 1(2) (1990), pp229-232.\t" }], "Reference", CellTags->"Ref:Luttrell1990"], Cell[TextData[{ "[7]\tGoodhill G J, ", StyleBox["Correlations, competition, and optimality: modelling the \ development of topography and ocular dominance", FontSlant->"Italic"], ", CSRP 226 (1992), Sussex University." }], "Reference", CellTags->"Ref:Goodhill1992"] }, Open ]] }, Open ]] }, FrontEndVersion->"5.0 for Microsoft Windows", ScreenRectangle->{{0, 1280}, {0, 941}}, WindowSize->{665.375, 641}, WindowMargins->{{307.25, Automatic}, {Automatic, 50}}, Magnification->1, StyleDefinitions -> "Report.nb" ] (******************************************************************* Cached data follows. If you edit this Notebook file directly, not using Mathematica, you must remove the line containing CacheID at the top of the file. The cache data will then be recreated when you save this file from within Mathematica. *******************************************************************) (*CellTagsOutline CellTagsIndex->{ "Sect:ObjectiveFunction"->{ Cell[14268, 434, 126, 6, 70, "Section", CellTags->"Sect:ObjectiveFunction"]}, "Eq:Objective"->{ Cell[14563, 449, 3234, 84, 70, "NumberedEquation", CellTags->{"Eq:Objective", "Ed:Change1"}]}, "Ed:Change1"->{ Cell[14563, 449, 3234, 84, 70, "NumberedEquation", CellTags->{"Eq:Objective", "Ed:Change1"}]}, "Eq:Independence"->{ Cell[24377, 734, 1427, 40, 70, "NumberedEquation", CellTags->"Eq:Independence"]}, "Ed:Change2"->{ Cell[26146, 789, 1173, 34, 70, "Text", CellTags->"Ed:Change2"]}, "Eq:LateralInhibition"->{ Cell[32644, 984, 2883, 74, 70, "NumberedEquation", CellTags->{"Eq:LateralInhibition", "Ed:Change3"}]}, "Ed:Change3"->{ Cell[32644, 984, 2883, 74, 70, "NumberedEquation", CellTags->{"Eq:LateralInhibition", "Ed:Change3"}]}, "Eq:Leakage"->{ Cell[40161, 1207, 1468, 40, 70, "NumberedEquation", CellTags->"Eq:Leakage"]}, "Sect:SimplifyObjectiveFunction"->{ Cell[44703, 1343, 147, 6, 70, "Section", CellTags->"Sect:SimplifyObjectiveFunction"]}, "Eq:ObjectiveSimplified"->{ Cell[45039, 1359, 3083, 74, 70, "NumberedEquation", CellTags->"Eq:ObjectiveSimplified"]}, "Eq:ObjectiveD2"->{ Cell[64442, 1877, 1942, 47, 70, "NumberedEquation", CellTags->"Eq:ObjectiveD2"]}, "Sect:DifferentiateObjectiveFunction"->{ Cell[69289, 2021, 151, 5, 70, "Section", CellTags->"Sect:DifferentiateObjectiveFunction"]}, "Eq:DerivativeXprime"->{ Cell[87129, 2488, 2493, 66, 70, "NumberedEquation", CellTags->"Eq:DerivativeXprime"]}, "Ed:Change5"->{ Cell[94983, 2704, 919, 26, 70, "Text", CellTags->"Ed:Change5"]}, "Eq:DerivativeWB"->{ Cell[96528, 2757, 5689, 139, 70, "NumberedEquation", CellTags->{"Eq:DerivativeWB", "Ed:Change4"}]}, "Ed:Change4"->{ Cell[96528, 2757, 5689, 139, 70, "NumberedEquation", CellTags->{"Eq:DerivativeWB", "Ed:Change4"}]}, "Sect:SpecialCases"->{ Cell[102850, 2923, 116, 6, 70, "Section", CellTags->"Sect:SpecialCases"]}, "Ed:Change6"->{ Cell[108331, 3100, 4346, 114, 70, "Text", CellTags->"Ed:Change6"]}, "Ref:Luttrell1994a"->{ Cell[114922, 3286, 462, 17, 70, "Reference", CellTags->"Ref:Luttrell1994a"]}, "Ref:Luttrell1996"->{ Cell[115387, 3305, 211, 6, 70, "Reference", CellTags->"Ref:Luttrell1996"]}, "Ref:Luttrell1994b"->{ Cell[115601, 3313, 513, 17, 70, "Reference", CellTags->"Ref:Luttrell1994b"]}, "Ref:LindeBuzoGray1980"->{ Cell[116117, 3332, 244, 6, 70, "Reference", CellTags->"Ref:LindeBuzoGray1980"]}, "Ref:Kohonen1984"->{ Cell[116364, 3340, 198, 6, 70, "Reference", CellTags->"Ref:Kohonen1984"]}, "Ref:Luttrell1990"->{ Cell[116565, 3348, 433, 15, 70, "Reference", CellTags->"Ref:Luttrell1990"]}, "Ref:Goodhill1992"->{ Cell[117001, 3365, 277, 7, 70, "Reference", CellTags->"Ref:Goodhill1992"]} } *) (*CellTagsIndex CellTagsIndex->{ {"Sect:ObjectiveFunction", 117973, 3393}, {"Eq:Objective", 118081, 3396}, {"Ed:Change1", 118204, 3399}, {"Eq:Independence", 118332, 3402}, {"Ed:Change2", 118442, 3405}, {"Eq:LateralInhibition", 118545, 3408}, {"Ed:Change3", 118676, 3411}, {"Eq:Leakage", 118807, 3414}, {"Sect:SimplifyObjectiveFunction", 118933, 3417}, {"Eq:ObjectiveSimplified", 119060, 3420}, {"Eq:ObjectiveD2", 119182, 3423}, {"Sect:DifferentiateObjectiveFunction", 119317, 3426}, {"Eq:DerivativeXprime", 119446, 3429}, {"Ed:Change5", 119561, 3432}, {"Eq:DerivativeWB", 119659, 3435}, {"Ed:Change4", 119787, 3438}, {"Sect:SpecialCases", 119922, 3441}, {"Ed:Change6", 120025, 3444}, {"Ref:Luttrell1994a", 120128, 3447}, {"Ref:Luttrell1996", 120240, 3450}, {"Ref:Luttrell1994b", 120351, 3453}, {"Ref:LindeBuzoGray1980", 120468, 3456}, {"Ref:Kohonen1984", 120582, 3459}, {"Ref:Luttrell1990", 120691, 3462}, {"Ref:Goodhill1992", 120802, 3465} } *) (*NotebookFileOutline Notebook[{ Cell[CellGroupData[{ Cell[1776, 53, 26, 0, 70, "Section 1"], Cell[CellGroupData[{ Cell[1827, 57, 39, 0, 70, "Subsection"], Cell[1869, 59, 413, 14, 70, "Text"], Cell[2285, 75, 295, 10, 70, "Text"], Cell[2583, 87, 2075, 64, 70, "Text"], Cell[4661, 153, 1598, 49, 70, "Text"], Cell[6262, 204, 4358, 120, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[10669, 330, 60, 0, 70, "Title"], Cell[10732, 332, 127, 4, 70, "Author"], Cell[10862, 338, 185, 4, 70, "Text"], Cell[11050, 344, 166, 3, 70, "Text"], Cell[11219, 349, 482, 9, 70, "Abstract"], Cell[CellGroupData[{ Cell[11726, 362, 78, 4, 70, "Section 1"], Cell[11807, 368, 1392, 27, 70, "Text"], Cell[13202, 397, 1029, 32, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[14268, 434, 126, 6, 70, "Section", CellTags->"Sect:ObjectiveFunction"], Cell[14397, 442, 163, 5, 70, "Text"], Cell[14563, 449, 3234, 84, 70, "NumberedEquation", CellTags->{"Eq:Objective", "Ed:Change1"}], Cell[17800, 535, 6422, 192, 70, "Text"], Cell[24225, 729, 149, 3, 70, "Text"], Cell[24377, 734, 1427, 40, 70, "NumberedEquation", CellTags->"Eq:Independence"], Cell[25807, 776, 336, 11, 70, "Text"], Cell[26146, 789, 1173, 34, 70, "Text", CellTags->"Ed:Change2"], Cell[27322, 825, 1122, 34, 70, "Text"], Cell[28447, 861, 1741, 46, 70, "NumberedEquation"], Cell[30191, 909, 2450, 73, 70, "Text"], Cell[32644, 984, 2883, 74, 70, "NumberedEquation", CellTags->{"Eq:LateralInhibition", "Ed:Change3"}], Cell[35530, 1060, 4013, 123, 70, "Text"], Cell[39546, 1185, 612, 20, 70, "Text"], Cell[40161, 1207, 1468, 40, 70, "NumberedEquation", CellTags->"Eq:Leakage"], Cell[41632, 1249, 3034, 89, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[44703, 1343, 147, 6, 70, "Section", CellTags->"Sect:SimplifyObjectiveFunction"], Cell[44853, 1351, 183, 6, 70, "Text"], Cell[45039, 1359, 3083, 74, 70, "NumberedEquation", CellTags->"Eq:ObjectiveSimplified"], Cell[48125, 1435, 6630, 188, 70, "Text"], Cell[54758, 1625, 294, 11, 70, "Text"], Cell[55055, 1638, 7897, 192, 70, "NumberedEquation"], Cell[62955, 1832, 1484, 43, 70, "Text"], Cell[64442, 1877, 1942, 47, 70, "NumberedEquation", CellTags->"Eq:ObjectiveD2"], Cell[66387, 1926, 2865, 90, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[69289, 2021, 151, 5, 70, "Section", CellTags->"Sect:DifferentiateObjectiveFunction"], Cell[69443, 2028, 465, 11, 70, "Text"], Cell[69911, 2041, 16116, 405, 70, "NumberedEquation"], Cell[86030, 2448, 1096, 38, 70, "Text"], Cell[87129, 2488, 2493, 66, 70, "NumberedEquation", CellTags->"Eq:DerivativeXprime"], Cell[89625, 2556, 501, 17, 70, "Text"], Cell[90129, 2575, 4487, 113, 70, "NumberedEquation"], Cell[94619, 2690, 361, 12, 70, "Text"], Cell[94983, 2704, 919, 26, 70, "Text", CellTags->"Ed:Change5"], Cell[95905, 2732, 620, 23, 70, "Text"], Cell[96528, 2757, 5689, 139, 70, "NumberedEquation", CellTags->{"Eq:DerivativeWB", "Ed:Change4"}], Cell[102220, 2898, 593, 20, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[102850, 2923, 116, 6, 70, "Section", CellTags->"Sect:SpecialCases"], Cell[102969, 2931, 146, 3, 70, "Text"], Cell[CellGroupData[{ Cell[103140, 2938, 151, 8, 70, "Subsection"], Cell[103294, 2948, 4865, 137, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[108196, 3090, 132, 8, 70, "Subsection"], Cell[108331, 3100, 4346, 114, 70, "Text", CellTags->"Ed:Change6"], Cell[112680, 3216, 1802, 50, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[114531, 3272, 30, 0, 70, "Section"], Cell[114564, 3274, 289, 5, 70, "Text"] }, Open ]], Cell[CellGroupData[{ Cell[114890, 3284, 29, 0, 70, "Section"], Cell[114922, 3286, 462, 17, 70, "Reference", CellTags->"Ref:Luttrell1994a"], Cell[115387, 3305, 211, 6, 70, "Reference", CellTags->"Ref:Luttrell1996"], Cell[115601, 3313, 513, 17, 70, "Reference", CellTags->"Ref:Luttrell1994b"], Cell[116117, 3332, 244, 6, 70, "Reference", CellTags->"Ref:LindeBuzoGray1980"], Cell[116364, 3340, 198, 6, 70, "Reference", CellTags->"Ref:Kohonen1984"], Cell[116565, 3348, 433, 15, 70, "Reference", CellTags->"Ref:Luttrell1990"], Cell[117001, 3365, 277, 7, 70, "Reference", CellTags->"Ref:Goodhill1992"] }, Open ]] }, Open ]] } ] *) (******************************************************************* End of Mathematica Notebook file. *******************************************************************)