(************** Content-type: application/mathematica ************** CreatedBy='Mathematica 5.0' Mathematica-Compatible Notebook This notebook can be used with any Mathematica-compatible application, such as Mathematica, MathReader or Publicon. The data for the notebook starts with the line containing stars above. To get the notebook into a Mathematica-compatible application, do one of the following: * Save the data starting with the line of stars above into a file with a name ending in .nb, then open the file inside the application; * Copy the data starting with the line of stars above to the clipboard, then use the Paste menu command inside the application. Data for notebooks contains only printable 7-bit ASCII and can be sent directly in email or through ftp in text mode. Newlines can be CR, LF or CRLF (Unix, Macintosh or MS-DOS style). NOTE: If you modify the data for this notebook not in a Mathematica- compatible application, you must delete the line below containing the word CacheID, otherwise Mathematica-compatible applications may try to use invalid cache data. For more information on notebooks and Mathematica-compatible applications, contact Wolfram Research: web: http://www.wolfram.com email: info@wolfram.com phone: +1-217-398-0700 (U.S.) Notebook reader applications are available free of charge from Wolfram Research. *******************************************************************) (*CacheID: 232*) (*NotebookFileLineBreakTest NotebookFileLineBreakTest*) (*NotebookOptionsPosition[ 140524, 4243]*) (*NotebookOutlinePosition[ 151074, 4550]*) (* CellTagsIndexPosition[ 148788, 4475]*) (*WindowFrame->Normal*) Notebook[{ Cell[CellGroupData[{ Cell["Notes", "Section 1"], Cell[CellGroupData[{ Cell["Editorial Changes", "Subsection"], Cell["\<\ References extracted from bibliography at end of book, and then appended to \ end of paper (for completeness).\ \>", "Text"], Cell["American spelling changed to British spelling.", "Text"], Cell["\<\ Chapter number 6 removed from section numbers, equation numbers, and figure \ numbers.\ \>", "Text"], Cell["\<\ Equation numbers in the appendix are renumbered using the same numbering \ sequence as those in the main text.\ \>", "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["PROBLEM", ButtonData:>"Ed:Problem2", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "This is wrong. I did not assign copyright to CUP." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["PROBLEM", ButtonData:>"Ed:Problem1", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`\(y\^\[Prime]\)\^\[DoublePrime]\)]], "\" doesn't look right. There is no symbol for triple prime." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change1", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{"1", "/", RowBox[{\(\[Sum]\+\(\(y\^\[Prime]\)\^\[DoublePrime] \[Element] \ \[ScriptCapitalN](y\^\[DoublePrime])\)\), RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", \(\(y\^\[Prime]\)\^\[DoublePrime]\)}], ")"}]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox["1", RowBox[{\(\[Sum]\+\(\(y\^\[Prime]\)\^\[DoublePrime] \[Element] \ \[ScriptCapitalN](y\^\[DoublePrime])\)\), RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", \(\(y\^\[Prime]\)\^\[DoublePrime]\)}], ")"}]}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change2", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"(", RowBox[{ StyleBox["1", FontWeight->"Plain"], "/", "M"}], ")"}], \(\[Sum]\+\(y\^\[DoublePrime] \[Element] \(\ \[ScriptCapitalN]\^\(-1\)\)(y\^\[Prime])\)\((\[CenterEllipsis])\)\)}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ StyleBox["1", FontWeight->"Plain"], "M"], \(\[Sum]\+\(y\^\[DoublePrime] \[Element] \ \(\[ScriptCapitalN]\^\(-1\)\)(y\^\[Prime])\)\((\[CenterEllipsis])\)\)}], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change3", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`\[Sum]\_\(\(y\^\[Prime]\)\^\[DoublePrime] \[Element] \ \[ScriptCapitalN](y\^\[DoublePrime])\)\%\(y\^\[DoublePrime] \[Element] \(\ \[ScriptCapitalN]\^\(-1\)\)(y\^\[Prime])\)\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`\[Sum]\+\(\(y\^\[DoublePrime] \[Element] \(\ \[ScriptCapitalN]\^\(-1\)\)(y\^\[Prime])\)\[IndentingNewLine]\(\(y\^\[Prime]\)\ \^\[DoublePrime] \[Element] \[ScriptCapitalN](y\^\[DoublePrime])\)\)\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change4", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`1/M\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`1\/M\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change5", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`\((a/M\ q\_0)\)\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`a\/\(M\ q\_0\)\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change6", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`\(-\((1/M\ q\_0)\)\) \((1/N\^2)\)\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`\(1\/\(M\ q\_0\)\) 1\/N\^2\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change7", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change8", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change9", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change14", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change16", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change17", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change20", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change21", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change23", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change25", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change27", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change28", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "Figure moved to a more appropriate place in the text." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change10", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`N/2\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`N\/2\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change11", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`1/a\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`1\/a\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change12", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`2 \[Pi]/k\_0\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`\(2 \[Pi]\)\/k\_0\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change13", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change22", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change26", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`exp(\(-1\)/2)\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`exp(\(-\(1\/2\)\))\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change15", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`2 \[Pi]/0.74\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`\(2 \[Pi]\)\/0.74\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change18", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"Figures 4 and 5\" changed to \"Figure 4 and Figure 5\" to accommodate \ two hyperlinks." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change19", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"Figures 7 and 8\" changed to \"Figure 7 and Figure 8\" to accommodate \ two hyperlinks." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change24", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"Figure 8\" changed to \"Figure 9\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change29", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["1", FontWeight->"Plain"], StyleBox["/", FontWeight->"Plain"], "M"}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ StyleBox["1", FontWeight->"Plain"], "M"], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change30", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["w", FontWeight->"Bold", FontSlant->"Plain"], "(", "y", ")"}], "\[CenterDot]", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["w", FontWeight->"Bold", FontSlant->"Plain"], "(", "y", ")"}], ".", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change31", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{\(\[PartialD]D\), "/", RowBox[{"\[PartialD]", RowBox[{"b", "(", StyleBox["y", FontSlant->"Italic"], ")"}]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[\(\[PartialD]D\), RowBox[{"\[PartialD]", RowBox[{"b", "(", StyleBox["y", FontSlant->"Italic"], ")"}]}]], TraditionalForm]]], "\", and \"", Cell[BoxData[ FormBox[ RowBox[{\(\[PartialD]D\), "/", RowBox[{"\[PartialD]", RowBox[{ StyleBox["w", FontWeight->"Bold", FontSlant->"Plain"], "(", "y", ")"}], "\[NegativeThinSpace]"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[\(\[PartialD]D\), RowBox[{"\[PartialD]", RowBox[{ StyleBox["w", FontWeight->"Bold", FontSlant->"Plain"], "(", "y", ")"}], "\[NegativeThinSpace]"}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change32", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "Equation reformatted to display on one line." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change33", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{\(\[PartialD]D\), "/", RowBox[{"\[PartialD]", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[\(\[PartialD]D\), RowBox[{"\[PartialD]", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}]}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change34", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], StyleBox["/", FontSlant->"Italic"], "\[Delta]"}], " ", "log", " ", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{"\[Delta]", " ", "log", " ", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}]}]], TraditionalForm]]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change35", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`\[Sum]\_\(\(y\^\[Prime]\)\^\[DoublePrime] \[Element] \ \[ScriptCapitalN](y\^\[DoublePrime])\)\%\(y\^\[DoublePrime] \[Element] \(\ \[ScriptCapitalN]\^\(-1\)\)(z)\)\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`\[Sum]\+\(\(y\^\[DoublePrime] \[Element] \(\ \[ScriptCapitalN]\^\(-1\)\)(z)\)\[IndentingNewLine]\(\(y\^\[Prime]\)\^\ \[DoublePrime] \[Element] \[ScriptCapitalN](y\^\[DoublePrime])\)\)\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change36", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ \(TraditionalForm\`\((1/N\^2)\)\)]], "\" changed to \"", Cell[BoxData[ \(TraditionalForm\`1\/N\^2\)]], "\"." }], "Text"], Cell[TextData[{ Cell[BoxData[ FormBox[ ButtonBox["TYPO", ButtonData:>"Ed:Change37", Active->True, ButtonStyle->"Hyperlink"], TextForm]]], "\"", Cell[BoxData[ FormBox[ RowBox[{\(\[Sum]\+\(y = 1\)\%M\), RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["y", FontWeight->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}]}], TraditionalForm]]], "\" changed to \"", Cell[BoxData[ FormBox[ RowBox[{\(\[Sum]\+\(y = 1\)\%M\), RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}]}], TraditionalForm]]], "\"." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["\<\ The Emergence of Dominance Stripes and Orientation Maps in a Network of \ Firing Neurons\ \>", "Title"], Cell["STEPHEN P. LUTTRELL", "Author"], Cell["\<\ This paper appeared in Proceedings of the 2nd Workshop on Information Theory \ and the Brain, Newquay, 101-121, 1996\ \>", "Text"], Cell[TextData[{ StyleBox["Information Theory and the Brain", FontSlant->"Italic"], ", edited by Roland Beddeley, Peter Hancock, and Peter F\[ODoubleDot]ldi\ \[AAcute]k. Copyright \[Copyright] 1999 Cambridge University Press. Printed \ in the United States of America. All rights reserved." }], "Text", CellTags->"Ed:Problem2"], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Introduction" }], "Section 1"], Cell[TextData[{ "This chapter addresses the problem of training a self-organising neural \ network on images derived from multiple sources; this type of network \ potentially may be used to model the behaviour of the mammalian visual cortex \ (for a review of neural network models of the visual cortex see Swindale (", ButtonBox["1996", ButtonData:>"Ref:Swindale1996", ButtonStyle->"Hyperlink"], "). The network that will be considered is a soft encoder which transforms \ its input vector into a posterior probability over various possible classes \ (i.e. alternative possible interpretations of the input vector). This encoder \ will be optimised so that its posterior probability is able to retain as much \ information as possible about its input vector, as measured in the minimum \ mean square reconstruction error (i.e. ", Cell[BoxData[ \(TraditionalForm\`L\_2\)]], " error) sense (Luttrell, ", ButtonBox["1994a", ButtonData:>"Ref:Luttrell1994a", ButtonStyle->"Hyperlink"], ", ", ButtonBox["1997c", ButtonData:>"Ref:Luttrell1997c", ButtonStyle->"Hyperlink"], ")." }], "Text"], Cell[TextData[{ "In the special case where the optimisation is performed over the space of \ all possible soft encoders, the optimum solution is a hard encoder (i.e. it \ is a \"winner-take-all\" network, in which only one of the output neurons is \ active) which is an optimal vector quantiser (VQ) of the type described in \ Linde et al. (", ButtonBox["1980", ButtonData:>"Ref:LindeBuzoGray1980", ButtonStyle->"Hyperlink"], "), for encoding the input vector with minimum ", Cell[BoxData[ \(TraditionalForm\`L\_2\)]], " error. A more general case is where the output of the soft encoder is \ deliberately damaged by the effects of a noise process. This type of noisy \ encoder leads to an optimal self-organising map (SOM) for encoding the input \ vector with minimum ", Cell[BoxData[ \(TraditionalForm\`L\_2\)]], " error, which is closely related to the well-known Kohonen map (Kohonen, \ ", ButtonBox["1984", ButtonData:>"Ref:Kohonen1984", ButtonStyle->"Hyperlink"], ")." }], "Text"], Cell[TextData[{ "The soft encoder network that is discussed in this chapter turns out to \ have many of the emergent properties that are observed in the mammalian \ visual cortex, such as dominance stripes and orientation maps. It will \ therefore be referred to as a Visual COrtex Network (VICON). It differs from \ other visual cortex models (see the review in Swindale (", ButtonBox["1996", ButtonData:>"Ref:Swindale1996", ButtonStyle->"Hyperlink"], ")) because it uses Bayesian methods to analyse the information contained \ in sets of neural firing events, where the neuron inputs are high-dimensional \ images rather than vectors in a low-dimensional abstract space. Also the \ network structure is derived from first principles, rather than \ reverse-engineered from observations of the structure of the visual cortex." }], "Text"], Cell[TextData[{ "The layout of this chapter is as follows. In ", ButtonBox["Section", ButtonData:>"Sect:2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2"], " the network objective function is presented (in the Appendix its \ derivatives are presented and interpreted). In ", ButtonBox["Section", ButtonData:>"Sect:3", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:3"], " the concepts of dominance stripes and orientation maps are explained. In \ ", ButtonBox["Section", ButtonData:>"Sect:4", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:4"], " the results of computer simulations are presented, where the effects of \ varying several parameters are explored. Both one- and two-dimensional \ retinae are considered, and in each of these cases both single retinae and \ pairs of retinae are considered." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Theory" }], "Section", CellTags->"Sect:2"], Cell["\<\ This section summarises the theory of a two-layer VICON. The network \ objective function is introduced, and its first-order perturbation expansion \ is interpreted.\ \>", "Text"], Cell[TextData[{ "The derivatives of the objective function with respect to its parameters \ are presented and interpreted in the ", ButtonBox["Appendix", ButtonData:>"Sect:Appendix", ButtonStyle->"Hyperlink"], "." }], "Text"], Cell[CellGroupData[{ Cell["Objective Function", "Subsection"], Cell[TextData[{ "For a two-layer network with ", Cell[BoxData[ \(TraditionalForm\`M\)]], " output neurons, the network objective function is given by (Luttrell, ", ButtonBox["1997c", ButtonData:>"Ref:Luttrell1997c", ButtonStyle->"Hyperlink"], ")" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{"D", "=", RowBox[{"2", RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontSlant->"Italic"], "=", "1"}], "M"], RowBox[{ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], SuperscriptBox[ RowBox[{"\[LeftBracketingBar]", RowBox[{"\[LeftBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}]}], "\[RightBracketingBar]"}], "\[RightBracketingBar]"}], "2"]}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->"Eq:1"], Cell[TextData[{ "where the probability ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " that neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], " fires first is given by" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], StyleBox["=", FontWeight->"Plain"], RowBox[{ FractionBox[ StyleBox["1", FontWeight->"Plain"], "M"], RowBox[{\(\[Sum]\+\(y\^\[Prime] = 1\)\%M\), RowBox[{\(Pr(y \[VerticalSeparator] y\^\[Prime])\), RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", \(y\^\[Prime]\)}], ")"}], RowBox[{\(\[Sum]\+\(y\^\[DoublePrime] \[Element] \(\ \[ScriptCapitalN]\^\(-1\)\)(y\^\[Prime])\)\), FractionBox["1", RowBox[{\(\[Sum]\+\(\(y\^\[Prime]\)\^\[DoublePrime] \ \[Element] \[ScriptCapitalN](y\^\[DoublePrime])\)\), RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", \(\(y\^\[Prime]\)\^\ \[DoublePrime]\)}], ")"}]}]]}]}]}]}]}], TraditionalForm]], "NumberedEquation",\ CellTags->{"Ed:Problem1", "Eq:2"}], Cell[TextData[{ "where ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " is the network input vector, ", Cell[BoxData[ FormBox[ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], TraditionalForm]]], " is the raw firing rate of neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], " in response to input ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " (which is proportional to the likelihood that input ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " is produced by neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], " when it is run \"in reverse\" as a generative model, rather than as a \ recognition model), ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalN](y)\)]], " is the ", StyleBox["local", FontSlant->"Italic"], " neighbourhood of neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], " (which is assumed to contain at least neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], ") and ", Cell[BoxData[ \(TraditionalForm\`\(\[ScriptCapitalN]\^\(-1\)\)(y)\)]], " is the inverse neighbourhood of neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], " defined as ", Cell[BoxData[ \(TraditionalForm\`\(\[ScriptCapitalN]\^\(-1\)\)( y\^\[Prime]) \[Congruent] {y\^\[Prime] \[VerticalSeparator] y \[Element] \[ScriptCapitalN](y\^\[Prime])}\)]], ". The term ", Cell[BoxData[ FormBox[ FractionBox["1", RowBox[{\(\[Sum]\+\(\(y\^\[Prime]\)\^\[DoublePrime] \[Element] \ \[ScriptCapitalN](y\^\[DoublePrime])\)\), RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", \(\(y\^\[Prime]\)\^\[DoublePrime]\)}], ")"}]}]], TraditionalForm]]], " is the lateral inhibition factor that derives from the neighbourhood of \ neuron ", Cell[BoxData[ \(TraditionalForm\`y\^\[DoublePrime]\)]], ", which gives rise to a contribution to the lateral inhibition factor for \ all neurons ", Cell[BoxData[ \(TraditionalForm\`y\^\[Prime]\)]], " in the neighbourhood of ", Cell[BoxData[ \(TraditionalForm\`y\^\[DoublePrime]\)]], " in the average ", Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ StyleBox["1", FontWeight->"Plain"], "M"], \(\[Sum]\+\(y\^\[DoublePrime] \[Element] \ \(\[ScriptCapitalN]\^\(-1\)\)(y\^\[Prime])\)\((\[CenterEllipsis])\)\)}], TraditionalForm]]], ". Thus the overall lateral inhibition factor acting on neuron ", Cell[BoxData[ \(TraditionalForm\`y\^\[Prime]\)]], " is derived ", StyleBox["locally", FontSlant->"Italic"], " from those neurons ", Cell[BoxData[ \(TraditionalForm\`\(y\^\[Prime]\)\^\[DoublePrime]\)]], " that lie in the set ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalN](\(\[ScriptCapitalN]\^\(-1\)\)( y\^\[Prime]))\)]], ", which is the union of the neighbourhoods ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalN](y)\)]], ", where ", Cell[BoxData[ \(TraditionalForm\`y \[Element] \(\[ScriptCapitalN]\^\(-1\)\)( y\^\[Prime])\)]], ". " }], "Text", CellTags->{"Ed:Change1", "Ed:Change2"}], Cell[TextData[{ "The information that is available at the output of the network (as encoded \ in ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], ") has been selectively damaged, so that neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], " fires (rather than the originally intended neuron ", Cell[BoxData[ \(TraditionalForm\`y\^\[Prime]\)]], ") with probability ", Cell[BoxData[ \(TraditionalForm\`Pr(y \[VerticalSeparator] y\^\[Prime])\)]], "; this is known as \"probability leakage\". In this paper it will be \ assumed that ", Cell[BoxData[ \(TraditionalForm\`Pr(y \[VerticalSeparator] y\^\[Prime])\)]], " is chosen to allow only ", StyleBox["local", FontSlant->"Italic"], " probability leakage (i.e. in a network where the ", Cell[BoxData[ \(TraditionalForm\`y\)]], " index is one-dimensional, ", Cell[BoxData[ \(TraditionalForm\`Pr(y \[VerticalSeparator] y\^\[Prime])\)]], " would be a rapidly decaying function of ", Cell[BoxData[ \(TraditionalForm\`\[LeftBracketingBar]y - y\^\[Prime]\[RightBracketingBar]\)]], "), which leads to topographic ordering of neuron properties (Kohonen, ", ButtonBox["1984", ButtonData:>"Ref:Kohonen1984", ButtonStyle->"Hyperlink"], "). Finally, the network attempts to reconstruct its input vector by making \ use of the set of reference vectors ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}], ",", \(y = 1\), ",", "2", ",", "\[CenterEllipsis]", ",", "M"}], TraditionalForm]]], "." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell["Interpretation of the Objective Function", "Subsection"], Cell[TextData[{ "In order to interpret the objective function ", Cell[BoxData[ \(TraditionalForm\`D\)]], " in ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:1"], ", a first order perturbation expansion of the expression for ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:2"], " will be derived. For simplicity, assume that all of the ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalN](y)\)]], " and ", Cell[BoxData[ \(TraditionalForm\`\(\[ScriptCapitalN]\^\(-1\)\)(y)\)]], " are the same size ", Cell[BoxData[ \(TraditionalForm\`N\)]], " (e.g. a translation invariant neighbourhood structure with periodic \ boundary conditions). Expand ", Cell[BoxData[ FormBox[ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], TraditionalForm]]], " about ", Cell[BoxData[ \(TraditionalForm\`q\_0\)]], ", define ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"\[CapitalDelta]", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], "=", RowBox[{ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], "-", \(q\_0\)}]}], TraditionalForm]]], ", drop terms of order ", Cell[BoxData[ FormBox[ SuperscriptBox[ RowBox[{"\[CapitalDelta]", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], "2"], TraditionalForm]]], " and higher (see Luttrell (", ButtonBox["1997a", ButtonData:>"Ref:Luttrell1997a", ButtonStyle->"Hyperlink"], ") for the details), to obtain" }], "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], "\[TildeTilde]", RowBox[{\(1\/M\), "+", RowBox[{\(1\/\(M\ q\_0\)\), RowBox[{\(\[Sum]\+\(y\^\[Prime] = 1\)\%M\), RowBox[{\(Pr(y \[VerticalSeparator] y\^\[Prime])\), RowBox[{"(", RowBox[{ RowBox[{"\[CapitalDelta]", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", \(y\^\[Prime]\)}], ")"}], "-", RowBox[{\(1\/N\^2\), RowBox[{\(\[Sum]\+\(\(y\^\[DoublePrime] \[Element] \(\ \[ScriptCapitalN]\^\(-1\)\)( y\^\[Prime])\)\[IndentingNewLine]\(\(y\^\ \[Prime]\)\^\[DoublePrime] \[Element] \[ScriptCapitalN]( y\^\[DoublePrime])\)\)\), RowBox[{"\[CapitalDelta]", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", \(\(y\^\[Prime]\)\^\ \[DoublePrime]\)}], ")"}]}]}]}], ")"}]}]}]}]}]}], TraditionalForm]], "NumberedEquation", CellTags->{"Ed:Change3", "Eq:3"}], Cell[TextData[{ "which satisfies ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(\[Sum]\+\(y = 1\)\%M\), RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}]}], StyleBox["=", FontWeight->"Plain"], StyleBox["1", FontWeight->"Plain"]}], TraditionalForm]]], ", so that ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " has the correct global normalisation." }], "Text"], Cell[TextData[{ "The quadratic and higher order terms have been dropped from ", ButtonBox["equation", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:3"], ", so it gives a linear (in ", Cell[BoxData[ FormBox[ RowBox[{"\[CapitalDelta]", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], TraditionalForm]]], ") approximation to ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], ". This linearity means that, to this level of approximation, the \ properties of ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " can be completely analysed by investigating the effect of an isolated \ signal ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"\[CapitalDelta]", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], "=", \(a\ \[Delta]\_\(y, y\_0\)\)}], TraditionalForm]]], ". (i.e. an isolated peak in ", Cell[BoxData[ FormBox[ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], TraditionalForm]]], ", which is located at ", Cell[BoxData[ \(TraditionalForm\`y = y\_0\)]], "). This is essentially a Green's function analysis, in which the impulse \ reponse of a linear system is first derived, and then used subsequently to \ build up the overall response to an arbitrary input. In this case ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " is a sum of three pieces: (1) a constant background term ", Cell[BoxData[ \(TraditionalForm\`1\/M\)]], "; (2) an isolated leakage function peak ", Cell[BoxData[ \(TraditionalForm\`\(a\/\(M\ q\_0\)\) \(Pr( y \[VerticalSeparator] y\_0)\)\)]], " centred at ", Cell[BoxData[ \(TraditionalForm\`y = y\_0\)]], "; (3) minus the average (for ", Cell[BoxData[ \(TraditionalForm\`y \[Element] \[ScriptCapitalN](\(\[ScriptCapitalN]\^\ \(-1\)\)(y\_0))\)]], ") over a set of isolated leakage function peaks ", Cell[BoxData[ \(TraditionalForm\`\(1\/\(M\ q\_0\)\) \(1\/N\^2\) \(\[Sum]\+\(y\^\ \[Prime] \[Element] \(\[ScriptCapitalN]\^\(-1\)\)(y\_0)\)\(\[Sum]\+\(y\^\ \[DoublePrime] \[Element] \[ScriptCapitalN](y\^\[Prime])\)Pr( y \[VerticalSeparator] y\^\[DoublePrime])\)\)\)]], ". Term (1) is the constant ", StyleBox["free", FontSlant->"Italic"], " response of the network, and terms (2) and (3) combine to give the local \ excitatory plus longer-range inhibitory ", StyleBox["forced", FontSlant->"Italic"], " response of the network. Because it is assumed that ", Cell[BoxData[ \(TraditionalForm\`Pr(y \[VerticalSeparator] y\^\[Prime])\)]], " describes local probability leakage, and that ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalN](y)\)]], " is a local neighbourhood set, then the excitatory term (2) and the \ inhibitory term (3) are non-zero only over a local region. Since these two \ terms cancel out in the exact ", StyleBox["global", FontSlant->"Italic"], " normalisation property ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(\[Sum]\+\(y = 1\)\%M\), RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}]}], StyleBox["=", FontWeight->"Plain"], StyleBox["1", FontWeight->"Plain"]}], TraditionalForm]]], ", and since both terms are non-zero only near ", Cell[BoxData[ \(TraditionalForm\`y = y\_0\)]], ", this implies that there is an exact ", StyleBox["local", FontSlant->"Italic"], " normalisation property which holds over a local region that is \ sufficiently large to include all of the non-zero contributions from these \ two terms. The net effect is that the (positive) area of the excitatory peak \ cancels out the (negative) area of the surrounding inhibitory trough." }], "Text", CellTags->{"Ed:Change4", "Ed:Change6", "Ed:Change5"}], Cell[TextData[{ "If the input is not an isolated signal ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"\[CapitalDelta]", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], "\[NotEqual]", \(a\ \[Delta]\_\(y, y\_0\)\)}], TraditionalForm]]], ", then this local normalisation property is no longer exact, because of \ the contributions to ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " that arise from outside any local region that one might consider as a \ candidate for having a local normalisation property. However, this is an edge \ effect, which reduces as the size of the local region is increased; this \ ensures that there is an approximate local normalisation property. In \ summary, the approximate local normalisation property derives from the exact \ global normalisation property ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(\[Sum]\+\(y = 1\)\%M\), RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}]}], StyleBox["=", FontWeight->"Plain"], StyleBox["1", FontWeight->"Plain"]}], TraditionalForm]]], " together with the assumed local nature of ", Cell[BoxData[ \(TraditionalForm\`Pr(y \[VerticalSeparator] y\^\[Prime])\)]], " and ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalN](y)\)]], "." }], "Text"], Cell[TextData[{ "Competition between local excitatory connections and longer range \ inhibitory connections is common to many models of the visual cortex \ (Swindale, ", ButtonBox["1996", ButtonData:>"Ref:Swindale1996", ButtonStyle->"Hyperlink"], "). However, in this chapter the network objective function ", Cell[BoxData[ \(TraditionalForm\`D\)]], " introduces this effect indirectly, by first postulating a model in which \ the neurons fire independently (i.e. ", Cell[BoxData[ FormBox[ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], TraditionalForm]]], "), and then deriving an expression for the probability that a neuron fires \ first (i.e. ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], "), where the neurons can no longer be treated in isolation of each other \ (although they still fire independently). Thus the excitatory and inhibitory \ connections are a consequence of building a probabilistic description of the \ neural firing events, rather than having been directly modelled at the level \ of the neural firing events themselves. In summary, knowledge about the \ firing behaviour of a network (i.e. ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], ") can exhibit lateral inhibition (i.e. if ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ SubscriptBox[ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], "1"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " is large then ", Cell[BoxData[ FormBox[ RowBox[{\(\[Sum]\+\(y \[NotEqual] y\_1\)\), RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}]}], TraditionalForm]]], " is small), even if the neurons fire independently." }], "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Dominance Stripes and Orientation Maps" }], "Section", CellTags->"Sect:3"], Cell[TextData[{ "The purpose of this section is to discuss the nature of dominance stripes \ and orientation maps, and to present a simple picture that makes it clear \ what types of behaviour should be expected from a neural network that \ minimises the objective function in ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:1"], "." }], "Text"], Cell[CellGroupData[{ Cell["Very-Low-Resolution Input Images", "Subsection"], Cell[TextData[{ "The simplest situation is when there are two retinae, each of which senses \ independently a featureless scene, i.e. all the pixels in a retina sense the \ same brightness value, but the two brightnesses that the left and right \ retinae sense are independent of each other. This situation approximates what \ happens when the images projected onto the two retinae are derived from \ different areas of a single image, whose resolution is low enough that each \ retina approximately senses a featureless scene. This limits the input data \ to lying in a two-dimensional space ", Cell[BoxData[ \(TraditionalForm\`R\^2\)]], " . If these two featureless input images (i.e. left and right retinae) are \ then normalised so that the sum of left and right retina brightness is \ constrained to be constant, then the input data is projected down onto a \ one-dimensional space ", Cell[BoxData[ \(TraditionalForm\`R\^1\)]], ", which effectively becomes the ocularity dimension (i.e. the difference \ between the right and left eye responses)." }], "Text"], Cell[TextData[{ "The optimal network (i.e. the one that minimises ", Cell[BoxData[ \(TraditionalForm\`D\)]], ") would then be one in which each of the ", Cell[BoxData[ \(TraditionalForm\`M\)]], " neurons had an infinitely wide receptive field to \"see\" the whole of \ the featureless input image. The optimal weight vectors and biases must then \ be chosen to give the best encoding (i.e. it minimises the objective function \ ", Cell[BoxData[ \(TraditionalForm\`D\)]], ") of the ", Cell[BoxData[ \(TraditionalForm\`R\^1\)]], " space that is visible within these infinitely wide receptive fields. \ However, because of the limited receptive field size and output layer \ neighbourhood size, the neurons can at best cooperate a few at a time (this \ also depends on the size of the leakage neighbourhood). If the network \ properties are translation invariant, then minimising ", Cell[BoxData[ \(TraditionalForm\`D\)]], " leads in general to an optimal solution which fluctuates periodically \ with position across the network (Luttrell, ", ButtonBox["1994b", ButtonData:>"Ref:Luttrell1994b", ButtonStyle->"Hyperlink"], "), where each period typically contains a complete repertoire of the \ computing machinery that is needed to soft encode ", Cell[BoxData[ \(TraditionalForm\`R\^1\)]], "; this effect is called \"completeness\", and it is a characteristic \ emergent property of this type of neural network. This is the origin of \ dominance stripes." }], "Text"], Cell[TextData[{ "A dubious step in this argument is the use of a normalisation procedure on \ the input. However, if the input to this network is assumed to be the \ posterior probability computed by the output layer of another network of the \ same type, then an approximate local normalisation property would \ automatically hold; this follows from the form of the posterior probability \ in ", ButtonBox["equation", ButtonData:>"Eq:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:2"], ", which is discussed in ", ButtonBox["Section", ButtonData:>"Sect:2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2"], ". This assumption would be correct for all layers of a multilayer network \ (except the input layer), provided that each layer integrates its input over \ time, so that the firing events that it responds to have a chance to build up \ into an extended pattern of activity, which is proportional to the posterior \ probability. This property may be used to normalise the input from the two \ retinae as follows. First, a pair (left and right) of retinal images is \ mapped to a single image in which the input pixels are interleaved (e.g. in a \ chessboard fashion) in order to bring corresponding left and right retina \ pixels into the same local region. Second, these pixel values are identified \ as the raw firing rates of a set of neurons. Finally, the posterior \ probability for which neuron fires first is derived, which automatically has \ an approximate local normalisation property. This posterior probability will \ be used as the input to our soft encoder network. The preprocessed retinal \ images are anti-correlated, even though the raw retinal images are \ statistically independent. This anti-correlation arises because the \ approximate local normalisation property implies that if the posterior \ probability for a left-eye neuron to fire first is large, then the posterior \ probability for nearby right-eye neurons to fire must be small, and vice \ versa. These results are summarised in ", ButtonBox["Figure", ButtonData:>"Fig:1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:1"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig1.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:1", "Ed:Change7"}], Cell["\<\ Typical neural reference vectors for very-low-resolution input images. The \ ocularity dimension runs from (0, 1) to (1, 0). As a function of position \ across the network, the neuron properties fluctuate periodically back and \ forth along the ocularity dimension.\ \>", "Caption"] }, Closed]], Cell[CellGroupData[{ Cell["Low-Resolution Input Images", "Subsection"], Cell[TextData[{ "A natural generalisation of the above is to the case of \ not-quite-featureless input images. This could be brought about by gradually \ increasing the resolution of the input images until it is sufficient to \ reveal spatial detail on a size scale equal to the receptive field size. \ Instead of seeing a featureless input, each neuron would then see a \ brightness gradient within its receptive field. This could be interpreted by \ considering the low-order terms of a Taylor expansion of the input image \ about a point at the centre of the neuron's receptive field: the zeroth term \ is local average brightness (which lives on a one-dimensional line ", Cell[BoxData[ \(TraditionalForm\`R\^1\)]], "), and the two first-order terms are the local brightness gradient (which \ lives in a two-dimensional space ", Cell[BoxData[ \(TraditionalForm\`R\^2\)]], "). When normalisation is applied this reduces the space in which the two \ images live to ", Cell[BoxData[ \(TraditionalForm\`R\^1\[Times]R\^2\[Times]R\^2\)]], " (", Cell[BoxData[ \(TraditionalForm\`R\^1\)]], " from the zeroth-order Taylor term with normalisation taken into account, \ ", Cell[BoxData[ \(TraditionalForm\`R\^2\)]], " from the first-order Taylor terms, counted twice to deal with both \ retinae). Note that the normalisation removes only one degree of freedom; it \ does not apply separately to each term in the Taylor expansion." }], "Text"], Cell[TextData[{ "The ", Cell[BoxData[ \(TraditionalForm\`R\^1\)]], " from the zeroth-order Taylor term gives rise to ocular dominance stripes \ (as discussed above) which thus causes the left and right retinae to map to \ different stripe-shaped regions of the output layer. The remaining ", Cell[BoxData[ \(TraditionalForm\`R\^2\[Times]R\^2\)]], " then naturally splits into two contributions (left retina and right \ retina), each of which maps to the appropriate stripe. If the stripes did not \ separate the left and right retinae in the network output layer, then the ", Cell[BoxData[ \(TraditionalForm\`R\^2\[Times]R\^2\)]], " could not be split apart in this simple manner. Finally, since each \ ocular dominance stripe occupies a two-dimensional region of the output layer \ of the network, a direct mapping of the corresponding ", Cell[BoxData[ \(TraditionalForm\`R\^2\)]], " (which carries local brightness gradient information) to output space can \ be made. As in the case of dominance stripes alone, the limited receptive \ field size and output neuron neighbourhood size causes the neurons to \ cooperate only a few at a time, so that each local patch of neurons contains \ a complete mapping from ", Cell[BoxData[ \(TraditionalForm\`R\^2\)]], " to the two-dimensional output layer (see Luttrell (", ButtonBox["1994b", ButtonData:>"Ref:Luttrell1994b", ButtonStyle->"Hyperlink"], ") for a discussion of this point). These results are summarised in ", ButtonBox["Figure", ButtonData:>"Fig:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:2"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig2.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:2", "Ed:Change8"}], Cell[TextData[{ "Typical neural reference vectors for low-resolution input images. The pure \ oscillation back and forth along the ocularity dimension that occurred in ", ButtonBox["Figure", ButtonData:>"Fig:1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:1"], " develops to reveal some additional degrees of freedom, only one of which \ is represented here (it is perpendicular to the ocularity axis)." }], "Caption"], Cell[TextData[{ "If the amount of probability leakage is reduced then the oscillation back \ and forth along the dominance axis tends to be more like a square wave than a \ sine wave, in which case ", ButtonBox["Figure", ButtonData:>"Fig:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:2"], " becomes as shown in ", ButtonBox["Figure", ButtonData:>"Fig:3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:3"], ". This change occurs because the effect of probability leakage is to \ encourage topographic ordering of the neuron properties, so the less leakage \ there is, the less the neuron properties feel obliged to vary smoothly as a \ function of position across the network." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig3.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:3", "Ed:Change9"}], Cell["\<\ Typical neural reference vectors for low-resolution input images, where \ reduced leakage causes the ocularity to switch abruptly back and forth. The \ neural reference vectors are bunched near to the points (0, 1) and (1, 0), \ and explore the additional degree(s) of freedom at each end of the ocularity \ axis. In the extreme case, where the ocularity switches back and forth as a \ square wave, the neurons separate into two clusters, one of which responds \ only to the left retina's image and the other to the right retina's image. \ Furthermore, within each of these clusters, the neurons explore the additonal \ degree(s) of freedom that occur within the corresponding retina's image. Note \ only one such degree of freedom is represented here; it is perpendicular to \ the ocularity axis.\ \>", "Caption"], Cell["\<\ The problem of choosing appropriate receptive field and output layer \ neighbourhood sizes is non-trivial. In this paper these parameters have been \ determined by hand. More generally, these parameters must be determined \ adaptively from a training set, but this much more difficult problem will not \ be addressed here.\ \>", "Text"], Cell["\<\ The above arguments can be generalised to the case of input images with fine \ spatial structure (i.e. lots of high-order terms in the Taylor expansion of \ the image brightness are required). However, more and more neurons (per \ receptive field) are required in order to build a faithful mapping from input \ space to a two-dimensional representation in output space. For a given number \ of neurons (per receptive field) a saturation point will quickly be reached, \ where the least important detail (from the point of view of the objective \ function) is discarded, keeping only those properties of the input images \ that best preserve the ability of the neural network to reconstruct its own \ input with minimum Euclidean error (on average).\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell["Theoretical Prediction of Dominance Stripe Wavelength", "Subsection"], Cell[TextData[{ "In Luttrell (", ButtonBox["1997a", ButtonData:>"Ref:Luttrell1997a", ButtonStyle->"Hyperlink"], ") an estimate of the wavenumber ", Cell[BoxData[ \(TraditionalForm\`k\_0\)]], " of dominance stripes (in the limit of zero resolution, as discussed \ above) was obtained using the first-order perturbation theory of ", ButtonBox["Section", ButtonData:>"Sect:2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2"], ", and using neighbourhoods ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalN](y)\)]], " having a soft Gaussian profile with half-width ", Cell[BoxData[ \(TraditionalForm\`a\)]], " (which approximately corresponds to the ", Cell[BoxData[ \(TraditionalForm\`N\/2\)]], " that appears in ", ButtonBox["Section", ButtonData:>"Sect:2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2"], "), and leakage probability functions ", Cell[BoxData[ \(TraditionalForm\`Pr(y\^\[Prime] \[VerticalSeparator] y)\)]], " having a Gaussian profile with half-width ", Cell[BoxData[ \(TraditionalForm\`l\)]], ", to yield" }], "Text", CellTags->"Ed:Change10"], Cell[BoxData[ \(TraditionalForm\`k\_0 = \(1\/a\) \@\(log(\(2 a\^2\)\/l\^2 + 1)\)\)], \ "NumberedEquation", CellTags->"Eq:4"], Cell[TextData[{ "This result is not dependent on the training data, because the training \ data used had a very low resolution (as described above), and the receptive \ field and output layer neighbourhood sizes were fixed by hand. The wavenumber \ ", Cell[BoxData[ \(TraditionalForm\`k\_0\)]], " is proportional to ", Cell[BoxData[ \(TraditionalForm\`1\/a\)]], " times a logarithmic correction factor, so the inhibitory connection range \ (which is ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalO](a)\)]], ") determines the wavelength of the dominance stripes (which is ", Cell[BoxData[ \(TraditionalForm\`\(2 \[Pi]\)\/k\_0\)]], "). As the leakage range ", Cell[BoxData[ \(TraditionalForm\`l\)]], " increases, the wavenumber ", Cell[BoxData[ \(TraditionalForm\`k\_0\)]], " decreases slowly as expected, because the leakage forces neighbouring \ neuron properties to be more correlated than they would have been without \ leakage. However, the prediction in ", ButtonBox["equation", ButtonData:>"Eq:4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:4"], " breaks down as ", Cell[BoxData[ \(TraditionalForm\`l\[LongRightArrow]0\)]], ", because in order to derive ", Cell[BoxData[ \(TraditionalForm\`k\_0\)]], " the Gaussian profiles of ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalN](y)\)]], " and ", Cell[BoxData[ \(TraditionalForm\`Pr(y\^\[Prime] \[VerticalSeparator] y)\)]], " were assumed to be continuous, rather than sampled (Luttrell, ", ButtonBox["1997a", ButtonData:>"Ref:Luttrell1997a", ButtonStyle->"Hyperlink"], ")." }], "Text", CellTags->{"Ed:Change11", "Ed:Change12"}] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Simulations" }], "Section", CellTags->"Sect:4"], Cell["\<\ Two types of training data will be used: synthetic and natural. Synthetic \ data is used in order to demonstrate simple properties of VICON, without \ introducing extraneous detail to complicate the interpretation of the \ results. Natural data is used to remove any doubt that the neural network is \ capable of producing interesting and useful results when it encounters data \ that is more representative of what it might encounter in the real world.\ \>", "Text"], Cell["\<\ In this section dominance stripes are produced from a one-dimensional retina, \ and these results are generalised to a two-dimensional retina. In both cases \ both synthetic and natural image results are shown.\ \>", "Text"], Cell[CellGroupData[{ Cell["Dominance Stripes: the One-Dimensional Case", "Subsection"], Cell[TextData[{ "The purpose of the simulations that are presented in this section is to \ demonstrate the emergence of ocular dominance stripes in the simplest \ possible realistic case. The results will correspond to the situation that \ was outlined in ", ButtonBox["Figure", ButtonData:>"Fig:1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:1"], "." }], "Text"], Cell[CellGroupData[{ Cell["Featureless Training Data", "Subsubsection"], Cell["\<\ In this simulation the network is presented with very-low-resolution input \ images. In fact, the resolution is so low that each image is entirely \ featureless, so that all the neurons in a retina have the same input \ brightness, but the two retinae have independent input brightnesses. These \ input images are preprocessed by interleaving their pixels in a chessboard \ fashion, then normalised by processing them so that they look like the \ posterior probability computed by the output layer of another such network; \ the neighbourhood size used for this normalisation process was chosen to be \ the same as the network's own output layer neighbourhood size. The emergence \ of dominance stripes depends critically on this hard-wired preprocessing, \ which brings together signals from corresponding parts of the two retinae. \ More generally, this pattern of wiring should be allowed to emerge \ automatically when the network is optimised, but this problem will not be \ studied here.\ \>", "Text"], Cell[TextData[{ "In the first simulation the parameters used were: ", Cell[BoxData[ \(TraditionalForm\`network\ size = 30\)]], ", ", Cell[BoxData[ \(TraditionalForm\`receptive\ field\ size = 9\)]], ", ", Cell[BoxData[ \(TraditionalForm\`output\ layer\ neighbourhood\ size = 5\)]], " (centred on the source neuron), ", Cell[BoxData[ \(TraditionalForm\`leakage\ neighbourhood\ size = 5\)]], " (centred on the source neuron), ", Cell[BoxData[ \(TraditionalForm\`number\ of\ training\ updates = 2000\)]], ", ", Cell[BoxData[ \(TraditionalForm\`update\ step\ size = 0.01\)]], ". For each neuron the leakage probability had a Gaussian profile centred \ on the neuron, and the standard deviation was chosen as 1, to make the \ profile fall from 1 on the source neuron to ", Cell[BoxData[ \(TraditionalForm\`exp(\(-\(1\/2\)\))\)]], " on each of its two closest neighbours. The precise values of these \ parameters is not critical, although the period of the dominance stripes is \ largely determined by the output layer neighbourhood size (see ", ButtonBox["equation", ButtonData:>"Eq:4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:4"], ")." }], "Text", CellTags->"Ed:Change13"], Cell["\<\ At the edges of the network the output layer neighbourhoods are symmetrically \ truncated by the edge of the network; thus the neighbourhood of an edge \ neuron contains only itself. The leakage neighbourhood is asymmetrically \ truncated on the side which abuts the edge; thus the leakage neighbourhood of \ an edge neuron contains itself and the two neighbouring neurons away from the \ edge (assuming a leakage neighbourhood size of 5). The details of this \ neighbourhood prescription do not have a marked effect on the appearance of \ dominance stripes in one-dimensional simulations, because their period is \ determined mainly by the width of the output layer neighbourhoods. However, \ in the two-dimensional simulations to be described below, the anisotropic \ shape of the output layer neighbourhoods strongly influences the orientation \ of the two-dimensional dominance stripes. For consistency, the same \ neighbourhood prescription is used for both one-dimensional and \ two-dimensional simulations.\ \>", "Text"], Cell["\<\ The update scheme used was a crude gradient-following algorithm parameterised \ by three numbers which controlled the rate at which the weight vectors, \ biases and reference vectors were updated. These three numbers were \ continuously adjusted to ensure that the maximum rate of change (as measured \ over all the neurons in the network) of the length of each weight vector \ (divided by the receptive field size), and also the maximum rate of change of \ the absolute value of each bias, was always equal to the requested update \ step size; this prescription will adjust the parameter values until they move \ around in the neighbourhood of their optimum values. The reference vectors \ were controlled in a similar way to the weight vectors, except that they used \ three times the update step size, which made them more agile than the weights \ and biases they were trying to follow. A more sophisticated algorithm would \ allow the update step size to be reduced as the optimum parameter values were \ approached, but this has not been implemented in this study.\ \>", "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig4.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:4", "Ed:Change14"}], Cell["\<\ One-dimensional dominance stripes after training on synthetic data.\ \>", "Caption"], Cell[TextData[{ "The ocular dominance stripes that emerge from this simulation are shown in \ ", ButtonBox["Figure", ButtonData:>"Fig:4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:4"], ". The ocularity for a given neuron was estimated by computing the average \ of the absolute deviations (as measured with respect to the overall mean \ reference vector component value, which is zero for the zero-mean training \ data used here) of its reference vector components, both for the left retina \ and the right retina. This allows two plots to be drawn: average value of \ absolute deviations from the mean in left retina's receptive field as a \ function of position across the network, and similarly the right retina's \ receptive field." }], "Text"], Cell[TextData[{ "As can be seen in ", ButtonBox["Figure", ButtonData:>"Fig:4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:4"], ", these two curves are approximately periodic, and are in antiphase with \ each other; this corresponds to the situation shown in ", ButtonBox["Figure", ButtonData:>"Fig:1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:1"], ". The amplitude of the ocularity curves is less than the 0.5 that would be \ required for the end points of the ocularity dimension to be reached, because \ one of the effects of leakage is to introduce a type of elastic tension \ between the reference vectors that causes them to contract towards zero \ ocularity." }], "Text"], Cell[TextData[{ "Assuming that the effective neighbourhood Gaussian width is ", Cell[BoxData[ \(TraditionalForm\`a = 2\)]], " (i.e. the half-width of the output layer neighbourhood size) and the \ effective leakage Gaussian width is ", Cell[BoxData[ \(TraditionalForm\`l = 1\)]], " (i.e. the half-width of the leakage neighbourhood size), the dominance \ stripe wavenumber predicted by ", ButtonBox["equation", ButtonData:>"Eq:4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:4"], " is ", Cell[BoxData[ \(TraditionalForm\`k\_0 = \(1\/2\) \@\(log(8\/1 + 1)\) \[TildeTilde] 0.74\)]], ", so the dominance stripe wavelength is predicted to be approximately ", Cell[BoxData[ \(TraditionalForm\`\(2 \[Pi]\)\/0.74 = 8.5\)]], ", which is greater than the estimated value of 7 that is obtained from the \ results shown in ", ButtonBox["Figure", ButtonData:>"Fig:4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:4"], ". This discrepancy could be due to overestimating the size of either ", Cell[BoxData[ \(TraditionalForm\`a\)]], " or ", Cell[BoxData[ \(TraditionalForm\`1\)]], ", which is likely, since in the simulation both the neighbourhood and the \ leakage were implemented using only finite-sized windows." }], "Text", CellTags->"Ed:Change15"], Cell[TextData[{ "If the above simulation is continued for a further 2000 updates with a \ reduced leakage, by reducing the standard deviation of the Gaussian leakage \ profile from 1 to 0.5, then the ocular dominance curves become more like \ square waves than sine waves, as shown in ", ButtonBox["Figure", ButtonData:>"Fig:5", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:5"], "; this is similar to the type of situation that was shown in ", ButtonBox["Figure", ButtonData:>"Fig:3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:3"], ", except that the input images are featureless in this case." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig5.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:5", "Ed:Change16"}], Cell["\<\ One-dimensional square-wave dominance stripes after further training with \ reduced probability leakage on synthetic data.\ \>", "Caption"] }, Closed]], Cell[CellGroupData[{ Cell["Natural Training Data", "Subsubsection"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig6.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:6", "Ed:Change17"}], Cell["\<\ Brodatz texture image used as a natural training image. The correlation \ length of the texture structure is in the range 5-10 pixels.\ \>", "Caption"], Cell[TextData[{ ButtonBox["Figure", ButtonData:>"Fig:6", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:6"], " shows the Brodatz texture image (Brodatz, ", ButtonBox["1966", ButtonData:>"Ref:Brodatz1996", ButtonStyle->"Hyperlink"], ") that was used to generate a more realistic training set than was used in \ the synthetic simulations described above." }], "Text"], Cell[TextData[{ "The correlation length of this texture is comparable to the receptive \ field size (9 pixels) and the output layer neighbourhood size (5 pixels), so \ a simulation using one-dimensional training vectors extracted from this \ two-dimensional Brodatz image will effectively see very-low-resolution \ training data (i.e. each training vector occupies no more than one \ correlation length of the Brodatz image), and should thus respond \ approximately as described in ", ButtonBox["Figure", ButtonData:>"Fig:1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:1"], ". This Brodatz image was chosen because it has a simple structure, which \ is similar to spatially correlated noise. More complicated Brodatz images \ would make the results obtained in this paper more difficult to interpret." }], "Text"], Cell[TextData[{ "The results corresponding to ", ButtonBox["Figure", ButtonData:>"Fig:4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:4"], " and ", ButtonBox["Figure", ButtonData:>"Fig:5", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:5"], " for this image are shown in ", ButtonBox["Figure", ButtonData:>"Fig:7", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:7"], " and ", ButtonBox["Figure", ButtonData:>"Fig:8", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:8"], ", respectively." }], "Text", CellTags->{"Ed:Change18", "Ed:Change19"}], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig7.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:7", "Ed:Change20"}], Cell["\<\ One-dimensional dominance stripes after training on natural data.\ \>", "Caption"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig8.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:8", "Ed:Change21"}], Cell["\<\ One-dimensional square wave dominance stripes after further training with \ reduced probability leakage on natural data.\ \>", "Caption"], Cell["\<\ The featureless image and Brodatz image results are similar to each other, \ except that the depth of the ocularity fluctuations is somewhat less in the \ Brodatz image case, because in the Brodatz case the training data is not \ actually featureless within each receptive field. Intuitively, it can be seen \ that for a given network, any increase in the image structure will increase \ the effective dimensionality of the input space, which will cause the neuron \ parameters to spread out to fill that space; the reduced depth of the \ ocularity fluctuations is a side-effect of this phenomenon.\ \>", "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["Dominance Stripes: the Two-Dimensional Case", "Subsection"], Cell["\<\ The results of the previous simulations are extended to the case of \ two-dimensional networks. The training schedule(s) used in the simulations \ have not been optimised. Usually the update rate is chosen conservatively \ (i.e. smaller than it needs to be) to avoid possible numerical instabilities, \ and the number of training updates is chosen to be larger than it needs to be \ to ensure that convergence has occurred. It is highly likely that much more \ efficient training schedules could be found.\ \>", "Text"], Cell[CellGroupData[{ Cell["Featureless Training Data", "Subsubsection"], Cell[TextData[{ "The results that were presented in ", ButtonBox["Figure", ButtonData:>"Fig:4", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:4"], " may readily be extended to the case of a two-dimensional network. The \ parameters used were: ", Cell[BoxData[ \(TraditionalForm\`network\ size = 100\[Times]100\)]], ", ", Cell[BoxData[ \(TraditionalForm\`receptive\ field\ size = 3\[Times]3\)]], " (which is artificially small to allow the simulation to run faster), \ output layer ", Cell[BoxData[ \(TraditionalForm\`neighbourhood\ size = 5\[Times]5\)]], " (centred on the source neuron), ", Cell[BoxData[ \(TraditionalForm\`leakage\ neighbourhood\ size = 3\[Times]3\)]], " (centred on the source neuron), ", Cell[BoxData[ \(TraditionalForm\`number\ of\ training\ updates = 24, 000\)]], " (dominance stripes develop quickly, so far fewer than 24,000 training \ updates could be used), ", Cell[BoxData[ \(TraditionalForm\`update\ step\ size = 0.001\)]], ". For each neuron the leakage probability had a Gaussian profile centred \ on the neuron, and the standard deviations were chosen as ", Cell[BoxData[ \(TraditionalForm\`1\[Times]1\)]], ", to make the profile fall from 1 on the source neuron to ", Cell[BoxData[ \(TraditionalForm\`exp(\(-\(1\/2\)\))\)]], " on each of its four closest neighbours. Apart from the different \ parameter values, the simulation was conducted in precisely the same way as \ in the one-dimensional case, and the results for ocular dominance are shown \ in ", ButtonBox["Figure", ButtonData:>"Fig:9", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:9"], ", where ocularity has been quantised as a binary-valued quantity solely \ for display purposes." }], "Text", CellTags->{"Ed:Change22", "Ed:Change24"}], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig9.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:9", "Ed:Change23"}], Cell["\<\ Two-dimensional dominance stripes after training on synthetic data. These \ results show the characteristic striped structure that is familiar from \ experiments on the mammalian visual cortex, and the dominance stripes run \ perpendicular to the boundary, as expected.\ \>", "Caption"], Cell[TextData[{ "As in the one-dimensional case studied above, the output layer \ neighbourhoods are truncated symmetrically, and the leakage neighbourhoods \ are truncated asymmetrically, at the edges of the network. Whilst this had \ little effect on the results that were obtained in the one-dimensional case, \ in the two-dimensional case the shape of the output layer neighbourhoods \ determines the direction in which the dominance stripes run. The symmetric \ truncation of the output layer neighbourhoods is used to force these \ neighbourhoods to become highly anisotropic near the edges of the network, \ where they become one-dimensional neighbourhoods running parallel to the edge \ of the network. This type of neighbourhood leads to dominance stripes that \ run perpendicular to the edge of the network, as observed in ", ButtonBox["Figure", ButtonData:>"Fig:9", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:9"], " (and in the mammalian visual cortex). If a weaker anisotropy were used \ (such as would be the case if the output layer neighbourhood truncation \ scheme were asymmetric), then ideally the dominance stripes should still run \ perpendicular to the edge of the network, but simulations show that the \ network parameters are then more likely to become trapped in a suboptimal \ configuration where the dominance stripes do not run everywhere perpendicular \ to the edge of the network." }], "Text"], Cell[TextData[{ "By counting the stripes as they pass through the boundary in ", ButtonBox["Figure", ButtonData:>"Fig:9", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:9"], ", the dominance stripe wavelength is estimated as 7. The theoretically \ predicted wavelength prediction is 8.5 (as in the one-dimensional case, where \ the same parameter values were used). Possible sources of this discrepency \ are discussed below." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell["Natural Training Data", "Subsubsection"], Cell[TextData[{ "The simulation, whose results were shown in ", ButtonBox["Figure", ButtonData:>"Fig:9", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:9"], ", may be repeated using the Brodatz image training set shown in ", ButtonBox["Figure", ButtonData:>"Fig:6", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:6"], ", to yield the results shown in ", ButtonBox["Figure", ButtonData:>"Fig:10", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:10"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig10.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:10", "Ed:Change25"}], Cell[TextData[{ "Two-dimensional dominance stripes after training on natural data. These \ results are not quite as stripe-like as the results in", " ", ButtonBox["Figure", ButtonData:>"Fig:9", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:9"], ",", " because in the Brodatz case the training data is not actually featureless \ within each receptive field." }], "Caption"], Cell["\<\ The correlation length of the Brodatz image (approximately 10 pixels) implies \ an upper limit on the receptive field and output layer neighbourhood sizes, \ because if these are too large then each neuron will erroneously attempt to \ process signals that derive from independent correlation areas of the input \ image. These sizes have been chosen by hand, so that each neuron responds \ mainly to signals that derive from a single correlation area. The period of \ the dominance stripes is thus approximately the correlation length of the \ Brodatz image.\ \>", "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["Orientation Maps", "Subsection"], Cell[TextData[{ "The purpose of the simulations presented in this section is to demonstrate \ the emergence of orientation maps in the simplest possible realistic case. In \ the case of two retinae, the results would correspond to the situation \ outlined in ", ButtonBox["Figure", ButtonData:>"Fig:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:2"], " (or, at least, a higher-dimensional version of ", ButtonBox["Figure", ButtonData:>"Fig:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:2"], ")." }], "Text"], Cell[CellGroupData[{ Cell["Orientation Map (One Retina)", "Subsubsection"], Cell[TextData[{ "In this simulation the parameters used were: ", Cell[BoxData[ \(TraditionalForm\`network\ size = 30\[Times]30\)]], ", ", Cell[BoxData[ \(TraditionalForm\`receptive\ field\ size = 17\[Times]17\)]], ", ", Cell[BoxData[ \(TraditionalForm\`output\ layer\ neighbourhood\ size = 9\[Times]9\)]], " (centred on the source neuron), ", Cell[BoxData[ \(TraditionalForm\`leakage\ neighbourhood\ size = 3\[Times]3\)]], " (centred on the source neuron), ", Cell[BoxData[ \(TraditionalForm\`number\ of\ training\ updates = 24, 000\)]], ", ", Cell[BoxData[ \(TraditionalForm\`update\ step\ size = 0.01\)]], ". For each neuron the leakage probability had a Gaussian profile centred \ on the neuron, and the standard deviations were chosen as ", Cell[BoxData[ \(TraditionalForm\`1\[Times]1\)]], ", to make the profile fall from 1 on the source neuron to ", Cell[BoxData[ \(TraditionalForm\`exp(\(-\(1\/2\)\))\)]], " on each of its four closest neighbours. The output layer neighbourhoods \ were truncated symmetrically, and the leakage neighbourhoods were truncated \ asymmetrically, at the edges of the network." }], "Text", CellTags->"Ed:Change26"], Cell[TextData[{ "Note that both the receptive field size and the output layer neighbourhood \ size are substantially larger than in the two-dimensional dominance stripe \ simulations, because many more neurons are required in order to allow \ orientation maps to develop than to allow dominance stripes to develop; in \ fact it would be preferable to use even larger sizes than were used here. To \ limit the computer run time this meant that the overall size of the neural \ network had to be reduced from ", Cell[BoxData[ \(TraditionalForm\`100\[Times]100\)]], " to ", Cell[BoxData[ \(TraditionalForm\`30\[Times]30\)]], ". The training set was the Brodatz texture image in ", ButtonBox["Figure", ButtonData:>"Fig:6", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:6"], ". The results are shown in ", ButtonBox["Figure", ButtonData:>"Fig:11", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:11"], "." }], "Text"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig11.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:11", "Ed:Change27"}], Cell["\<\ Orientation map after training on natural data. The receptive fields have \ been gathered together in a montage. There is a clear swirl-like pattern that \ is characteristic of orientation maps. Each local clockwise or anticlockwise \ swirl typically circulates around an unoriented region. The contrast in this \ figure has been manually adjusted in order to make the fine details easier to \ see.\ \>", "Caption"] }, Closed]], Cell[CellGroupData[{ Cell["Using the Orientation Map", "Subsubsection"], Cell[TextData[Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/itbrain96/fig12.gif"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]]], "NumberedFigure", TextAlignment->Center, CellTags->{"Fig:12", "Ed:Change28"}], Cell["\<\ Typical input, output and reconstruction produced by the orientation map. The \ output consists of a number of isolated \"activity bubbles\" of posterior \ probability, and the reconstruction is a low-resolution version of the \ original input.\ \>", "Caption"], Cell[TextData[{ "In ", ButtonBox["Figure", ButtonData:>"Fig:12", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:12"], " the orientation map network shown in ", ButtonBox["Figure", ButtonData:>"Fig:11", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:11"], " is used to encode and decode a typical input image. On the left of ", ButtonBox["Figure", ButtonData:>"Fig:12", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:12"], " the input image (i.e. ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], ") is shown, in the centre of ", ButtonBox["Figure", ButtonData:>"Fig:12", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:12"], " the corresponding output (i.e. its posterior probability ", Cell[BoxData[ FormBox[ RowBox[{"Pr", "(", RowBox[{"y", "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], ")"}], TraditionalForm]]], ") produced by the orientation map in ", ButtonBox["Figure", ButtonData:>"Fig:11", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:11"], " is shown, and on the right of ", ButtonBox["Figure", ButtonData:>"Fig:12", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:12"], " the corresponding reconstruction (i.e. ", Cell[BoxData[ FormBox[ RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox["y", FontSlant->"Italic"], "=", "1"}], "M"], RowBox[{ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}]}]}], TraditionalForm]]], ") is shown." }], "Text"], Cell[TextData[{ "The form of the output in the centre of ", ButtonBox["Figure", ButtonData:>"Fig:12", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedFigure", "Fig:12"], " is familiar as a type of \"sparse coding\" of the input, where only a \ small fraction of the neurons participate in encoding a given input (this \ type of transformation of the input is central to the work that was reported \ in Webber (", ButtonBox["1994", ButtonData:>"Ref:Webber1994", ButtonStyle->"Hyperlink"], ")). This type of encoding is very convenient because it has effectively \ transformed the input into a small number of constituents each of which \ corresponds to a small patch of neural activity (an \"activity bubble\"), \ rather than transforming the input into a representation where the output \ activity is spread over all of the neurons, which would not be easily \ interpretable as arising from a small number of constituents." }], "Text"], Cell["\<\ The reconstruction has a lower resolution than the input because there are \ insufficient neurons to faithfully record all the information that is \ required to reconstruct the input exactly (e.g. probability leakage causes \ neighbouring neurons to have a correlated response, thus reducing the \ effective number of neurons that are available). The featureless region \ around the edge of the reconstruction is an edge effect, which occurs because \ fewer neurons (per unit area) contribute to the reconstruction near the edge \ of the input array. The quality of the reconstruction may be improved by \ increasing the size of the network (to reduce the proportion of the network \ that is distorted by the edge effect), or using more neurons per unit area of \ the image (to increase the resolution of the reconstruction).\ \>", "Text"] }, Closed]] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[TextData[{ CounterBox["Section"], " Conclusions" }], "Section"], Cell[TextData[{ "This chapter has shown that the theory of self-organising networks \ introduced in Luttrell (", ButtonBox["1994a", ButtonData:>"Ref:Luttrell1994a", ButtonStyle->"Hyperlink"], ") yields a class of self-organising neural networks (Luttrell, ", ButtonBox["1997c", ButtonData:>"Ref:Luttrell1997c", ButtonStyle->"Hyperlink"], ") which has many of the properties that are observed in the mammalian \ visual cortex (Swindale, ", ButtonBox["1996", ButtonData:>"Ref:Swindale1996", ButtonStyle->"Hyperlink"], "). This type of network will thus be called a VIsual COrtex Network \ (VICON). These neural networks differ from previous models of the visual \ cortex, insofar as they model the neuron behaviour in terms of their \ individual firing events, and also operate in the real space of input images. \ When the neural network structure (e.g. receptive ", "field size) parameters are chosen to match the correlation properties of \ the training data, dominance stripes and orientation maps emerge when the \ network is trained on a natural image (e.g. a Brodatz texture image)." }], "Text"], Cell["\<\ These results show that if this type of network is trained on data from \ multiple sources, then its internal parameters self-organise into the \ expected patterns, such as dominance stripes. Only one or two sources have \ been used in this study, but the same network objective function could be \ used if an arbitrary number of sources is used, and it is anticipated that it \ would lead to analogous results.\ \>", "Text"], Cell[TextData[{ "Although VICON has many emergent properties (such as dominance stripes and \ orientation maps) that are similar to observed properties of the mammalian \ visual cortex, VICON is nevertheless derived from the soft encoder theory in \ Luttrell (", ButtonBox["1994a", ButtonData:>"Ref:Luttrell1994a", ButtonStyle->"Hyperlink"], "), which is not a physiologically motivated theory. It is not yet clear \ whether VICON is a useful model of the visual cortex." }], "Text"], Cell[TextData[{ "The main limitation of the results that have been presented in this \ chapter is that the network structure is hand-crafted (e.g. the receptive \ field and output layer neighbourhood sizes). This restriction has been \ deliberately imposed for simplicity; it is not a fundamental limitation of \ the approach. For instance, it is possible to do more sophisticated \ simulations in which the network objective function is also optimised with \ respect to both the receptive field and output layer neighbourhood sizes \ (Luttrell, ", ButtonBox["1997b", ButtonData:>"Ref:Luttrell1997b", ButtonStyle->"Hyperlink"], ")." }], "Text"] }, Closed]], Cell[CellGroupData[{ Cell["Acknowledgements", "Section"], Cell["\<\ I thank Chris Webber for many useful discussions that we had during the \ course of this research.\ \>", "Text"] }, Closed]], Cell[CellGroupData[{ Cell["APPENDIX", "Section", CellTags->"Sect:Appendix"], Cell["\<\ The derivatives of the objective function with respect to its parameters are \ presented and interpreted.\ \>", "Text"], Cell[CellGroupData[{ Cell["Derivatives of the Objective Function", "Subsection"], Cell[TextData[{ "In order to minimise the network objective function in ", ButtonBox["equation", ButtonData:>"Eq:1", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:1"], " (using the exact expression for ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:2"], ", rather than its approximation in ", ButtonBox["equation", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:3"], ") its derivatives must be calculated. First of all, define some convenient \ notation (Luttrell, ", ButtonBox["1997c", ButtonData:>"Ref:Luttrell1997c", ButtonStyle->"Hyperlink"], "):" }], "Text"], Cell[BoxData[ FormBox[GridBox[{ { RowBox[{ SubscriptBox["L", RowBox[{"y", ",", StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], "\[Congruent]", RowBox[{"Pr", "(", RowBox[{ StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], "y"}], ")"}]}], RowBox[{ SubscriptBox["P", RowBox[{"y", StyleBox[",", FontWeight->"Plain"], StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], "\[Congruent]", RowBox[{"Pr", "(", RowBox[{ RowBox[{ StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[VerticalSeparator]", StyleBox["x", FontWeight->"Bold"]}], ";", "y"}], ")"}], "\[Congruent]", FractionBox[ RowBox[{ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", \(y\^\[Prime]\)}], ")"}], \(\[Delta]\_\(y\^\[Prime] \[Element] \ \[ScriptCapitalN](y)\)\)}], RowBox[{\(\[Sum]\+\(y\^\[DoublePrime] \[Element] \ \[ScriptCapitalN](y)\)\), RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", \(y\^\[DoublePrime]\)}], ")"}]}]]}]}, { RowBox[{\(p\_y\), "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", \(\(\[ScriptCapitalN]\^\(-1\)\)(y)\)}]], SubscriptBox["P", RowBox[{ StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ",", "y"}]]}]}], RowBox[{\(\((\(L\^T\) p)\)\_y\), "\[Congruent]", RowBox[{\(\[Sum]\+\(y\^\[Prime] = 1\)\%M\), RowBox[{ SubscriptBox["L", RowBox[{ StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ",", "y"}]], SubscriptBox["p", StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}]}, { RowBox[{\(e\_y\), "\[Congruent]", SuperscriptBox[ RowBox[{"\[LeftBracketingBar]", RowBox[{"\[LeftBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", "y", ")"}]}], "\[RightBracketingBar]"}], "\[RightBracketingBar]"}], "2"]}], RowBox[{\(\((L\ e)\)\_y\), "\[Congruent]", RowBox[{\(\[Sum]\+\(y\^\[Prime] = 1\)\%M\), RowBox[{ SubscriptBox["L", RowBox[{"y", ",", StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], SubscriptBox["e", StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}]}, { RowBox[{\(\((P\ L\ e)\)\_y\), "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", \(\[ScriptCapitalN](y)\)}]], RowBox[{ SubscriptBox["P", RowBox[{"y", ",", StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]}]], " ", SubscriptBox[\((L\ e)\), StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}], RowBox[{\(\((\(P\^T\) P\ L\ e)\)\_y\), "\[Congruent]", RowBox[{ SubscriptBox["\[Sum]", RowBox[{ StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], "\[Element]", \(\(\[ScriptCapitalN]\^\(-1\)\)(y)\)}]], RowBox[{ SubscriptBox["P", RowBox[{ StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"], ",", "y"}]], " ", SubscriptBox[\((P\ L\ e)\), StyleBox[ SuperscriptBox[ StyleBox["y", FontWeight->"Plain"], StyleBox["\[Prime]", FontWeight->"Plain"]], FontWeight->"Bold"]]}]}]}]} }], TraditionalForm]], "NumberedEquation", TextAlignment->AlignmentMarker, GridBoxOptions->{ColumnAlignments->{Left}}], Cell[TextData[{ "whence ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:2"], " may be written as ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], StyleBox["=", FontWeight->"Plain"], RowBox[{ FractionBox[ StyleBox["1", FontWeight->"Plain"], "M"], \(\((\(L\^T\) p)\)\_y\)}]}], TraditionalForm]]], ", and the derivatives of ", Cell[BoxData[ \(TraditionalForm\`D\)]], " may be obtained in the form (Luttrell, ", ButtonBox["1997c", ButtonData:>"Ref:Luttrell1997c", ButtonStyle->"Hyperlink"], ")" }], "Text", CellTags->"Ed:Change29"], Cell[BoxData[{ FormBox[ RowBox[{ FractionBox[\(\[PartialD]D\), RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", "y", ")"}]], "\[AlignmentMarker]", "=", RowBox[{\(-\(4\/M\)\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], \(\((\(L\^T\) p)\)\_y\), RowBox[{"(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}]}], ")"}]}]}]}]}], TraditionalForm], "\n", FormBox[ RowBox[{ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{"\[Delta]", " ", "log", " ", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}]}]], "\[AlignmentMarker]", "=", RowBox[{\(2\/M\), RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], \((p\_y\ \((L\ e)\)\_y - \((\(P\^T\) P\ L\ e)\)\_y)\)}]}], TraditionalForm]}], "NumberedEquation", TextAlignment->AlignmentMarker, CellTags->"Eq:6"], Cell["\<\ Assume that the raw neuron firing rates may be modelled using a sigmoid \ function\ \>", "Text"], Cell[BoxData[ FormBox[ RowBox[{ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], "=", FractionBox["1", RowBox[{"1", "+", RowBox[{"exp", "(", RowBox[{ RowBox[{"-", RowBox[{ RowBox[{ StyleBox["w", FontWeight->"Bold", FontSlant->"Plain"], "(", "y", ")"}], ".", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]}], "-", \(b(y)\)}], ")"}]}]]}], TraditionalForm]], "NumberedEquation", CellTags->"Ed:Change30"], Cell[TextData[{ "then the two derivatives ", Cell[BoxData[ FormBox[ FractionBox[\(\[PartialD]D\), RowBox[{"\[PartialD]", RowBox[{"b", "(", StyleBox["y", FontSlant->"Italic"], ")"}]}]], TraditionalForm]]], " and ", Cell[BoxData[ FormBox[ FractionBox[\(\[PartialD]D\), RowBox[{"\[PartialD]", RowBox[{ StyleBox["w", FontWeight->"Bold", FontSlant->"Plain"], "(", "y", ")"}], "\[NegativeThinSpace]"}]], TraditionalForm]]], " may be obtained in the form" }], "Text", CellTags->"Ed:Change31"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[\(\[PartialD]D\), RowBox[{"\[PartialD]", RowBox[{"(", "\[NegativeThinSpace]", GridBox[{ {\(b(y)\)}, { RowBox[{ StyleBox["w", FontWeight->"Bold", FontSlant->"Plain"], "(", "y", ")"}]} }], "\[NegativeThinSpace]", ")"}]}]], "\[AlignmentMarker]", "=", RowBox[{\(2\/M\), RowBox[{"\[Integral]", RowBox[{ StyleBox[ RowBox[{ StyleBox["d", FontSlant->"Italic"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}]], " ", RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], " ", RowBox[{"(", RowBox[{\(p\_y\ \((L\ e)\)\_y\), "-", SubscriptBox[ RowBox[{"(", \(\(P\^T\) P\ L\ e\), StyleBox[")", FontWeight->"Bold"]}], "y"]}], ")"}], RowBox[{"(", RowBox[{"1", "-", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}]}], ")"}], RowBox[{"(", "\[NegativeThinSpace]", GridBox[{ {"1"}, { StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]} }], "\[NegativeThinSpace]", ")"}]}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, SpanMaxSize->Infinity, CellTags->"Ed:Change32"] }, Closed]], Cell[CellGroupData[{ Cell["Interpretation of the Derivatives of the Objective Function", \ "Subsection"], Cell[TextData[{ "In ", ButtonBox["equation", ButtonData:>"Eq:6", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:6"], ", ", Cell[BoxData[ FormBox[ FractionBox[\(\[PartialD]D\), RowBox[{"\[PartialD]", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}]}]], TraditionalForm]]], " may be interpreted as follows. Thus ", Cell[BoxData[ \(TraditionalForm\`D\)]], " is reduced by moving the reference vector ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}], TraditionalForm]]], " directly towards the input vector ", Cell[BoxData[ FormBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], TraditionalForm]]], " by an amount that is proportional to the probability ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " that neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], " fires first; all the reference vectors ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}], TraditionalForm]]], " (for ", Cell[BoxData[ \(TraditionalForm\`y = 1, 2, \[CenterEllipsis], M\)]], ") are simultaneously updated in this fashion. This type of update \ prescription is a soft version of the winner-take-all prescription proposed \ by Kohonen (", ButtonBox["1984", ButtonData:>"Ref:Kohonen1984", ButtonStyle->"Hyperlink"], "), and it similarly leads to topographic ordering of the reference \ vectors." }], "Text", CellTags->"Ed:Change33"], Cell[TextData[{ "In ", ButtonBox["equation", ButtonData:>"Eq:6", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:6"], ", ", Cell[BoxData[ FormBox[ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{"\[Delta]", " ", "log", " ", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}]}]], TraditionalForm]]], " may be most easily interpreted by replacing ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " in ", ButtonBox["equation", ButtonData:>"Eq:2", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:2"], " by the approximation in ", ButtonBox["equation", ButtonData:>"Eq:3", ButtonStyle->"Hyperlink"], " ", CounterBox["NumberedEquation", "Eq:3"], ". Thus" }], "Text", CellTags->"Ed:Change34"], Cell[BoxData[ FormBox[ RowBox[{ FractionBox[ StyleBox[ RowBox[{"\[Delta]", StyleBox["D", FontSlant->"Italic"]}]], RowBox[{"\[Delta]", " ", RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "z"}], ")"}]}]], "\[AlignmentMarker]", "\[TildeTilde]", RowBox[{\(2\/\(M\ q\_0\)\), RowBox[{"Pr", "(", StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], ")"}], RowBox[{\(\[Sum]\+\(y = 1\)\%M\), RowBox[{\((Pr( y \[VerticalSeparator] z) - \(1\/N\^2\) \(\[Sum]\+\(\(y\^\[DoublePrime] \ \[Element] \(\[ScriptCapitalN]\^\(-1\)\)(z)\)\[IndentingNewLine]\(\(y\^\ \[Prime]\)\^\[DoublePrime] \[Element] \ \[ScriptCapitalN](y\^\[DoublePrime])\)\)\[CapitalDelta]( y \[VerticalSeparator] \ \(y\^\[Prime]\)\^\[DoublePrime])\))\), SuperscriptBox[ RowBox[{"\[LeftBracketingBar]", RowBox[{"\[LeftBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}]}], "\[RightBracketingBar]"}], "\[RightBracketingBar]"}], "2"]}]}]}]}], TraditionalForm]], "NumberedEquation", TextAlignment->Left, CellTags->"Ed:Change35"], Cell[TextData[{ "Thus ", Cell[BoxData[ \(TraditionalForm\`D\)]], " is reduced by reducing ", Cell[BoxData[ FormBox[ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], TraditionalForm]]], " by an amount that is proportional to the sum of the following two terms:" }], "Text"], Cell[TextData[{ "1. ", Cell[BoxData[ FormBox[ RowBox[{\(\[Sum]\+\(y\^\[Prime] = 1\)\%M\), RowBox[{\(Pr(y\^\[Prime] \[VerticalSeparator] y)\), SuperscriptBox[ RowBox[{"\[LeftBracketingBar]", RowBox[{"\[LeftBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox[\(y\^\[Prime]\), FontSlant->"Italic"], ")"}]}], "\[RightBracketingBar]"}], "\[RightBracketingBar]"}], "2"]}]}], TraditionalForm]]], ": a local excitatory term, which requires that those neurons that have a \ larger (or smaller) than average contribution to the reconstruction error to \ reduce (or increase) their raw firing rate. The exact definition of this \ average is given below in the interpretation of the inhibitory term." }], "Text"], Cell[TextData[{ "2. ", Cell[BoxData[ FormBox[ RowBox[{\(-\(1\/N\^2\)\), RowBox[{ UnderoverscriptBox["\[Sum]", RowBox[{ StyleBox[\(y\^\[Prime]\), FontSlant->"Italic"], "=", "1"}], "M"], RowBox[{ RowBox[{"(", RowBox[{\(\[Sum]\+\(\(y\^\[DoublePrime] \[Element] \(\ \[ScriptCapitalN]\^\(-1\)\)( y)\)\[IndentingNewLine]\(\(y\^\[Prime]\)\^\ \[DoublePrime] \[Element] \[ScriptCapitalN](y\^\[DoublePrime])\)\)\), RowBox[{"Pr", "(", RowBox[{ StyleBox[\(y\^\[Prime]\), FontSlant->"Italic"], "\[VerticalSeparator]", \(\(y\^\[Prime]\)\^\ \[DoublePrime]\)}], ")"}]}], ")"}], SuperscriptBox[ RowBox[{"\[LeftBracketingBar]", RowBox[{"\[LeftBracketingBar]", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "-", RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox[\(y\^\[Prime]\), FontSlant->"Italic"], ")"}]}], "\[RightBracketingBar]"}], "\[RightBracketingBar]"}], "2"]}]}]}], TraditionalForm]]], ": a longer-range inhibitory term, which is the average value (as measured \ over the neighbourhood ", Cell[BoxData[ \(TraditionalForm\`\[ScriptCapitalN](\(\[ScriptCapitalN]\^\(-1\)\)( y\^\[Prime]))\)]], " of the reconstruction error." }], "Text", CellTags->"Ed:Change36"], Cell[TextData[{ "Overall, this update prescription ensures that the raw firing rates ", Cell[BoxData[ FormBox[ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], TraditionalForm]]], " and the reference vectors ", Cell[BoxData[ FormBox[ RowBox[{ SuperscriptBox[ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[Prime]"], "(", StyleBox["y", FontSlant->"Italic"], ")"}], TraditionalForm]]], " (for ", Cell[BoxData[ \(TraditionalForm\`y = 1, 2, \[CenterEllipsis], M\)]], ") are adjusted in such a way that the objective function ", Cell[BoxData[ \(TraditionalForm\`D\)]], " is reduced." }], "Text"], Cell[TextData[{ "Although the neurons fire independently, the update prescription involves \ local excitatory and longer-range inhibitory terms. These excitatory and \ inhibitory terms are not included in the underlying neural firing model. \ Rather, they arise as a side-effect of constructing an objective function for \ network optimisation that depends on the posterior probability ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " that neuron y fires first. The model used for ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], ", taken together with the normalisation property ", Cell[BoxData[ FormBox[ RowBox[{ RowBox[{\(\[Sum]\+\(y = 1\)\%M\), RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}]}], StyleBox["=", FontWeight->"Plain"], StyleBox["1", FontWeight->"Plain"]}], TraditionalForm]]], ", then automatically leads to local excitatory and longer-range inhibitory \ terms, as discussed in ", ButtonBox["Section", ButtonData:>"Sect:2", ButtonStyle->"Hyperlink"], " ", CounterBox["Section", "Sect:2"], ". These excitatory and inhibitory terms do not affect the raw firing rate \ ", Cell[BoxData[ FormBox[ RowBox[{"Q", "(", RowBox[{ StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"], "\[VerticalSeparator]", "y"}], ")"}], TraditionalForm]]], " of the neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], ", but they do affect the posterior probability ", Cell[BoxData[ FormBox[ RowBox[{ StyleBox["Pr", FontWeight->"Plain"], StyleBox["(", FontWeight->"Plain"], RowBox[{ StyleBox["y", FontWeight->"Plain", FontSlant->"Italic"], StyleBox["\[VerticalSeparator]", FontWeight->"Plain"], StyleBox["x", FontWeight->"Bold", FontSlant->"Plain"]}], StyleBox[")", FontWeight->"Plain"]}], TraditionalForm]]], " that neuron ", Cell[BoxData[ \(TraditionalForm\`y\)]], " fires first, which in turn affects the value of the network objective \ function ", Cell[BoxData[ \(TraditionalForm\`D\)]], ", and hence the update prescription." }], "Text", CellTags->"Ed:Change37"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell["References", "Section"], Cell[TextData[{ "Brodatz, P. (1966). ", StyleBox["Textures - a photographic album for artists and designers", FontSlant->"Italic"], ", Dover, New York." }], "Reference", CellTags->"Ref:Brodatz1996"], Cell[TextData[{ "Kohonen, T. (1984). ", StyleBox["Self-organisation and associative memory", FontSlant->"Italic"], ", Springer-Verlag, Berlin." }], "Reference", CellTags->"Ref:Kohonen1984"], Cell[TextData[{ "Linde, Y., Buzo. A. and Gray, R. (1980). An algorithm for vector quantiser \ design, ", StyleBox["IEEE Trans. COM", FontSlant->"Italic"], " ", StyleBox["28", FontWeight->"Bold"], ": 84-95." }], "Reference", CellTags->"Ref:LindeBuzoGray1980"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/bayessom/bayessom.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell, S. P. (1994a). A Bayesian analysis of self-organising maps, ", StyleBox["Neural Computation", FontSlant->"Italic"], " ", StyleBox["6", FontWeight->"Bold"], ": 767-794." }], "Reference", CellTags->"Ref:Luttrell1994a"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/pmd/pmd.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell, S. P. (1994b). The partitioned mixture distribution: an adaptive \ Bayesian network for low-level image processing, ", StyleBox["Proceedings of the IEE, Vision, Image and Signal Processing", FontSlant->"Italic"], " ", StyleBox["141", FontWeight->"Bold"], ":, 251-260." }], "Reference", CellTags->"Ref:Luttrell1994b"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/ieenn97/ieenn97b.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell, S. P. (1997a). Partitioned mixture distributions: a first order \ perturbation analysis, ", StyleBox["Proceedings of the 5th IEE International Conference on Artificial \ Neural Networks", FontSlant->"Italic"], ", pp. 280-284." }], "Reference", CellTags->"Ref:Luttrell1997a"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/combine/combine.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell, S. P. (1997b). Self-organisation of multiple winner-take-all \ neural networks, ", StyleBox["Connection Science", FontSlant->"Italic"], " ", StyleBox["9", FontWeight->"Bold"], ": 11-30." }], "Reference", CellTags->"Ref:Luttrell1997b"], Cell[TextData[{ Cell[TextData[{ " ", ButtonBox["OPEN", ButtonData:>{ URL[ "http://www.luttrell.org.uk/papers/manna95/manna95.nb"], None}, Active->True, ButtonStyle->"Hyperlink"], " " }]], "Luttrell, S. P. (1997c). A theory of self-organising neural networks, ", StyleBox["in", FontSlant->"Italic"], " S. Ellacott, J. Mason and I. Anderson (eds.), ", StyleBox["Mathematics of Neural Networks: Models, Algorithms and \ Applications", FontSlant->"Italic"], ", Kluwer." }], "Reference", CellTags->"Ref:Luttrell1997c"], Cell[TextData[{ "Swindale, N. (1996). The development of topography in the visual cortex: a \ review of models, ", StyleBox["Network", FontSlant->"Italic"], " ", StyleBox["7", FontWeight->"Bold"], ": 161-247." }], "Reference", CellTags->"Ref:Swindale1996"], Cell[TextData[{ "Webber C. (1994). Self-organisation of transformation-invariant detectors \ for constituents of perceptual patterns, ", StyleBox["Network", FontSlant->"Italic"], " ", StyleBox["5", FontWeight->"Bold"], ": 471-495." }], "Reference", CellTags->"Ref:Webber1994"] }, Closed]] }, Open ]] }, FrontEndVersion->"5.0 for Microsoft Windows", ScreenRectangle->{{0, 1280}, {0, 941}}, WindowToolbars->"EditBar", WindowSize->{665, 641}, WindowMargins->{{Automatic, 300}, {Automatic, 50}}, Magnification->1, StyleDefinitions -> "Report.nb" ] (******************************************************************* Cached data follows. If you edit this Notebook file directly, not using Mathematica, you must remove the line containing CacheID at the top of the file. The cache data will then be recreated when you save this file from within Mathematica. *******************************************************************) (*CellTagsOutline CellTagsIndex->{ "Ed:Problem2"->{ Cell[19776, 725, 336, 7, 46, "Text", CellTags->"Ed:Problem2"]}, "Sect:2"->{ Cell[24197, 844, 92, 5, 41, "Section", CellTags->"Sect:2"]}, "Eq:1"->{ Cell[25076, 881, 2240, 55, 70, "NumberedEquation", CellTags->"Eq:1"]}, "Ed:Problem1"->{ Cell[28017, 964, 1757, 48, 70, "NumberedEquation", CellTags->{"Ed:Problem1", "Eq:2"}]}, "Eq:2"->{ Cell[28017, 964, 1757, 48, 70, "NumberedEquation", CellTags->{"Ed:Problem1", "Eq:2"}]}, "Ed:Change1"->{ Cell[29777, 1014, 3722, 115, 70, "Text", CellTags->{"Ed:Change1", "Ed:Change2"}]}, "Ed:Change2"->{ Cell[29777, 1014, 3722, 115, 70, "Text", CellTags->{"Ed:Change1", "Ed:Change2"}]}, "Ed:Change3"->{ Cell[38644, 1296, 1927, 47, 70, "NumberedEquation", CellTags->{"Ed:Change3", "Eq:3"}]}, "Eq:3"->{ Cell[38644, 1296, 1927, 47, 70, "NumberedEquation", CellTags->{"Ed:Change3", "Eq:3"}]}, "Ed:Change4"->{ Cell[42021, 1393, 6192, 174, 70, "Text", CellTags->{"Ed:Change4", "Ed:Change6", "Ed:Change5"}]}, "Ed:Change6"->{ Cell[42021, 1393, 6192, 174, 70, "Text", CellTags->{"Ed:Change4", "Ed:Change6", "Ed:Change5"}]}, "Ed:Change5"->{ Cell[42021, 1393, 6192, 174, 70, "Text", CellTags->{"Ed:Change4", "Ed:Change6", "Ed:Change5"}]}, "Sect:3"->{ Cell[54429, 1750, 124, 5, 41, "Section", CellTags->"Sect:3"]}, "Fig:1"->{ Cell[59949, 1881, 290, 10, 70, "NumberedFigure", CellTags->{"Fig:1", "Ed:Change7"}]}, "Ed:Change7"->{ Cell[59949, 1881, 290, 10, 70, "NumberedFigure", CellTags->{"Fig:1", "Ed:Change7"}]}, "Fig:2"->{ Cell[63789, 1977, 290, 10, 70, "NumberedFigure", CellTags->{"Fig:2", "Ed:Change8"}]}, "Ed:Change8"->{ Cell[63789, 1977, 290, 10, 70, "NumberedFigure", CellTags->{"Fig:2", "Ed:Change8"}]}, "Fig:3"->{ Cell[65321, 2025, 290, 10, 70, "NumberedFigure", CellTags->{"Fig:3", "Ed:Change9"}]}, "Ed:Change9"->{ Cell[65321, 2025, 290, 10, 70, "NumberedFigure", CellTags->{"Fig:3", "Ed:Change9"}]}, "Ed:Change10"->{ Cell[67677, 2077, 1206, 40, 70, "Text", CellTags->"Ed:Change10"]}, "Eq:4"->{ Cell[68886, 2119, 131, 3, 70, "NumberedEquation", CellTags->"Eq:4"]}, "Ed:Change11"->{ Cell[69020, 2124, 1749, 50, 70, "Text", CellTags->{"Ed:Change11", "Ed:Change12"}]}, "Ed:Change12"->{ Cell[69020, 2124, 1749, 50, 70, "Text", CellTags->{"Ed:Change11", "Ed:Change12"}]}, "Sect:4"->{ Cell[70818, 2180, 97, 5, 41, "Section", CellTags->"Sect:4"]}, "Ed:Change13"->{ Cell[73230, 2241, 1287, 35, 70, "Text", CellTags->"Ed:Change13"]}, "Fig:4"->{ Cell[76656, 2312, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:4", "Ed:Change14"}]}, "Ed:Change14"->{ Cell[76656, 2312, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:4", "Ed:Change14"}]}, "Ed:Change15"->{ Cell[78611, 2370, 1397, 40, 70, "Text", CellTags->"Ed:Change15"]}, "Fig:5"->{ Cell[80705, 2433, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:5", "Ed:Change16"}]}, "Ed:Change16"->{ Cell[80705, 2433, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:5", "Ed:Change16"}]}, "Fig:6"->{ Cell[81234, 2455, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:6", "Ed:Change17"}]}, "Ed:Change17"->{ Cell[81234, 2455, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:6", "Ed:Change17"}]}, "Ed:Change18"->{ Cell[82970, 2506, 699, 31, 70, "Text", CellTags->{"Ed:Change18", "Ed:Change19"}]}, "Ed:Change19"->{ Cell[82970, 2506, 699, 31, 70, "Text", CellTags->{"Ed:Change18", "Ed:Change19"}]}, "Fig:7"->{ Cell[83672, 2539, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:7", "Ed:Change20"}]}, "Ed:Change20"->{ Cell[83672, 2539, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:7", "Ed:Change20"}]}, "Fig:8"->{ Cell[84061, 2555, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:8", "Ed:Change21"}]}, "Ed:Change21"->{ Cell[84061, 2555, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:8", "Ed:Change21"}]}, "Ed:Change22"->{ Cell[85851, 2603, 1894, 49, 70, "Text", CellTags->{"Ed:Change22", "Ed:Change24"}]}, "Ed:Change24"->{ Cell[85851, 2603, 1894, 49, 70, "Text", CellTags->{"Ed:Change22", "Ed:Change24"}]}, "Fig:9"->{ Cell[87748, 2654, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:9", "Ed:Change23"}]}, "Ed:Change23"->{ Cell[87748, 2654, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:9", "Ed:Change23"}]}, "Fig:10"->{ Cell[90958, 2744, 293, 10, 70, "NumberedFigure", CellTags->{"Fig:10", "Ed:Change25"}]}, "Ed:Change25"->{ Cell[90958, 2744, 293, 10, 70, "NumberedFigure", CellTags->{"Fig:10", "Ed:Change25"}]}, "Ed:Change26"->{ Cell[93021, 2813, 1233, 30, 70, "Text", CellTags->"Ed:Change26"]}, "Fig:11"->{ Cell[95270, 2875, 293, 10, 70, "NumberedFigure", CellTags->{"Fig:11", "Ed:Change27"}]}, "Ed:Change27"->{ Cell[95270, 2875, 293, 10, 70, "NumberedFigure", CellTags->{"Fig:11", "Ed:Change27"}]}, "Fig:12"->{ Cell[96081, 2901, 293, 10, 70, "NumberedFigure", CellTags->{"Fig:12", "Ed:Change28"}]}, "Ed:Change28"->{ Cell[96081, 2901, 293, 10, 70, "NumberedFigure", CellTags->{"Fig:12", "Ed:Change28"}]}, "Sect:Appendix"->{ Cell[104210, 3134, 56, 1, 70, "Section", CellTags->"Sect:Appendix"]}, "Ed:Change29"->{ Cell[114508, 3407, 1804, 63, 70, "Text", CellTags->"Ed:Change29"]}, "Eq:6"->{ Cell[116315, 3472, 2114, 59, 70, "NumberedEquation", CellTags->"Eq:6"]}, "Ed:Change30"->{ Cell[118541, 3538, 805, 23, 70, "NumberedEquation", CellTags->"Ed:Change30"]}, "Ed:Change31"->{ Cell[119349, 3563, 663, 21, 70, "Text", CellTags->"Ed:Change31"]}, "Ed:Change32"->{ Cell[120015, 3586, 1981, 52, 70, "NumberedEquation", CellTags->"Ed:Change32"]}, "Ed:Change33"->{ Cell[122119, 3646, 2578, 83, 70, "Text", CellTags->"Ed:Change33"]}, "Ed:Change34"->{ Cell[124700, 3731, 1571, 58, 70, "Text", CellTags->"Ed:Change34"]}, "Ed:Change35"->{ Cell[126274, 3791, 1916, 47, 70, "NumberedEquation", CellTags->"Ed:Change35"]}, "Ed:Change36"->{ Cell[129831, 3884, 1895, 46, 70, "Text", CellTags->"Ed:Change36"]}, "Ed:Change37"->{ Cell[132595, 3961, 3879, 117, 70, "Text", CellTags->"Ed:Change37"]}, "Ref:Brodatz1996"->{ Cell[136555, 4086, 210, 6, 70, "Reference", CellTags->"Ref:Brodatz1996"]}, "Ref:Kohonen1984"->{ Cell[136768, 4094, 201, 6, 70, "Reference", CellTags->"Ref:Kohonen1984"]}, "Ref:LindeBuzoGray1980"->{ Cell[136972, 4102, 278, 10, 70, "Reference", CellTags->"Ref:LindeBuzoGray1980"]}, "Ref:Luttrell1994a"->{ Cell[137253, 4114, 486, 19, 70, "Reference", CellTags->"Ref:Luttrell1994a"]}, "Ref:Luttrell1994b"->{ Cell[137742, 4135, 569, 19, 70, "Reference", CellTags->"Ref:Luttrell1994b"]}, "Ref:Luttrell1997a"->{ Cell[138314, 4156, 528, 17, 70, "Reference", CellTags->"Ref:Luttrell1997a"]}, "Ref:Luttrell1997b"->{ Cell[138845, 4175, 495, 19, 70, "Reference", CellTags->"Ref:Luttrell1997b"]}, "Ref:Luttrell1997c"->{ Cell[139343, 4196, 575, 19, 70, "Reference", CellTags->"Ref:Luttrell1997c"]}, "Ref:Swindale1996"->{ Cell[139921, 4217, 276, 10, 70, "Reference", CellTags->"Ref:Swindale1996"]}, "Ref:Webber1994"->{ Cell[140200, 4229, 296, 10, 70, "Reference", CellTags->"Ref:Webber1994"]} } *) (*CellTagsIndex CellTagsIndex->{ {"Ed:Problem2", 141200, 4261}, {"Sect:2", 141288, 4264}, {"Eq:1", 141371, 4267}, {"Ed:Problem1", 141471, 4270}, {"Eq:2", 141581, 4273}, {"Ed:Change1", 141697, 4276}, {"Ed:Change2", 141808, 4279}, {"Ed:Change3", 141919, 4282}, {"Eq:3", 142029, 4285}, {"Ed:Change4", 142145, 4288}, {"Ed:Change6", 142270, 4291}, {"Ed:Change5", 142395, 4294}, {"Sect:3", 142516, 4297}, {"Fig:1", 142602, 4300}, {"Ed:Change7", 142716, 4303}, {"Fig:2", 142825, 4306}, {"Ed:Change8", 142939, 4309}, {"Fig:3", 143048, 4312}, {"Ed:Change9", 143162, 4315}, {"Ed:Change10", 143277, 4318}, {"Eq:4", 143366, 4321}, {"Ed:Change11", 143465, 4324}, {"Ed:Change12", 143578, 4327}, {"Sect:4", 143686, 4330}, {"Ed:Change13", 143777, 4333}, {"Fig:4", 143867, 4336}, {"Ed:Change14", 143983, 4339}, {"Ed:Change15", 144099, 4342}, {"Fig:5", 144189, 4345}, {"Ed:Change16", 144305, 4348}, {"Fig:6", 144415, 4351}, {"Ed:Change17", 144531, 4354}, {"Ed:Change18", 144647, 4357}, {"Ed:Change19", 144759, 4360}, {"Fig:7", 144865, 4363}, {"Ed:Change20", 144981, 4366}, {"Fig:8", 145091, 4369}, {"Ed:Change21", 145207, 4372}, {"Ed:Change22", 145323, 4375}, {"Ed:Change24", 145436, 4378}, {"Fig:9", 145543, 4381}, {"Ed:Change23", 145659, 4384}, {"Fig:10", 145770, 4387}, {"Ed:Change25", 145887, 4390}, {"Ed:Change26", 146004, 4393}, {"Fig:11", 146095, 4396}, {"Ed:Change27", 146212, 4399}, {"Fig:12", 146324, 4402}, {"Ed:Change28", 146441, 4405}, {"Sect:Appendix", 146560, 4408}, {"Ed:Change29", 146659, 4411}, {"Eq:6", 146749, 4414}, {"Ed:Change30", 146851, 4417}, {"Ed:Change31", 146959, 4420}, {"Ed:Change32", 147055, 4423}, {"Ed:Change33", 147164, 4426}, {"Ed:Change34", 147261, 4429}, {"Ed:Change35", 147358, 4432}, {"Ed:Change36", 147467, 4435}, {"Ed:Change37", 147564, 4438}, {"Ref:Brodatz1996", 147666, 4441}, {"Ref:Kohonen1984", 147774, 4444}, {"Ref:LindeBuzoGray1980", 147888, 4447}, {"Ref:Luttrell1994a", 148005, 4450}, {"Ref:Luttrell1994b", 148118, 4453}, {"Ref:Luttrell1997a", 148231, 4456}, {"Ref:Luttrell1997b", 148344, 4459}, {"Ref:Luttrell1997c", 148457, 4462}, {"Ref:Swindale1996", 148569, 4465}, {"Ref:Webber1994", 148678, 4468} } *) (*NotebookFileOutline Notebook[{ Cell[CellGroupData[{ Cell[1776, 53, 26, 0, 70, "Section 1"], Cell[CellGroupData[{ Cell[1827, 57, 39, 0, 70, "Subsection"], Cell[1869, 59, 134, 3, 70, "Text"], Cell[2006, 64, 62, 0, 70, "Text"], Cell[2071, 66, 110, 3, 70, "Text"], Cell[2184, 71, 134, 3, 70, "Text"], Cell[2321, 76, 252, 8, 70, "Text"], Cell[2576, 86, 348, 11, 70, "Text"], Cell[2927, 99, 1178, 36, 70, "Text"], Cell[4108, 137, 865, 29, 70, "Text"], Cell[4976, 168, 691, 18, 70, "Text"], Cell[5670, 188, 333, 14, 70, "Text"], Cell[6006, 204, 355, 14, 70, "Text"], Cell[6364, 220, 385, 14, 70, "Text"], Cell[6752, 236, 2098, 74, 70, "Text"], Cell[8853, 312, 334, 14, 70, "Text"], Cell[9190, 328, 334, 14, 70, "Text"], Cell[9527, 344, 358, 14, 70, "Text"], Cell[9888, 360, 694, 26, 70, "Text"], Cell[10585, 388, 358, 14, 70, "Text"], Cell[10946, 404, 289, 9, 70, "Text"], Cell[11238, 415, 289, 9, 70, "Text"], Cell[11530, 426, 237, 8, 70, "Text"], Cell[11770, 436, 561, 22, 70, "Text"], Cell[12334, 460, 836, 30, 70, "Text"], Cell[13173, 492, 1343, 44, 70, "Text"], Cell[14519, 538, 244, 8, 70, "Text"], Cell[14766, 548, 989, 32, 70, "Text"], Cell[15758, 582, 1205, 40, 70, "Text"], Cell[16966, 624, 672, 18, 70, "Text"], Cell[17641, 644, 346, 14, 70, "Text"], Cell[17990, 660, 1438, 47, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[19477, 713, 113, 3, 175, "Title"], Cell[19593, 718, 37, 0, 66, "Author"], Cell[19633, 720, 140, 3, 46, "Text"], Cell[19776, 725, 336, 7, 46, "Text", CellTags->"Ed:Problem2"], Cell[CellGroupData[{ Cell[20137, 736, 78, 4, 70, "Section 1"], Cell[20218, 742, 1131, 25, 70, "Text"], Cell[21352, 769, 1032, 24, 70, "Text"], Cell[22387, 795, 852, 14, 70, "Text"], Cell[23242, 811, 918, 28, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[24197, 844, 92, 5, 41, "Section", CellTags->"Sect:2"], Cell[24292, 851, 189, 4, 70, "Text"], Cell[24484, 857, 239, 7, 70, "Text"], Cell[CellGroupData[{ Cell[24748, 868, 40, 0, 70, "Subsection"], Cell[24791, 870, 282, 9, 70, "Text"], Cell[25076, 881, 2240, 55, 70, "NumberedEquation", CellTags->"Eq:1"], Cell[27319, 938, 695, 24, 70, "Text"], Cell[28017, 964, 1757, 48, 70, "NumberedEquation", CellTags->{"Ed:Problem1", "Eq:2"}], Cell[29777, 1014, 3722, 115, 70, "Text", CellTags->{"Ed:Change1", "Ed:Change2"}], Cell[33502, 1131, 2284, 65, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[35823, 1201, 62, 0, 70, "Subsection"], Cell[35888, 1203, 2753, 91, 70, "Text"], Cell[38644, 1296, 1927, 47, 70, "NumberedEquation", CellTags->{"Ed:Change3", "Eq:3"}], Cell[40574, 1345, 1444, 46, 70, "Text"], Cell[42021, 1393, 6192, 174, 70, "Text", CellTags->{"Ed:Change4", "Ed:Change6", "Ed:Change5"}], Cell[48216, 1569, 2471, 68, 70, "Text"], Cell[50690, 1639, 3690, 105, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[54429, 1750, 124, 5, 41, "Section", CellTags->"Sect:3"], Cell[54556, 1757, 430, 12, 70, "Text"], Cell[CellGroupData[{ Cell[55011, 1773, 54, 0, 70, "Subsection"], Cell[55068, 1775, 1086, 19, 70, "Text"], Cell[56157, 1796, 1539, 35, 70, "Text"], Cell[57699, 1833, 2247, 46, 70, "Text"], Cell[59949, 1881, 290, 10, 70, "NumberedFigure", CellTags->{"Fig:1", "Ed:Change7"}], Cell[60242, 1893, 292, 5, 70, "Caption"] }, Closed]], Cell[CellGroupData[{ Cell[60571, 1903, 49, 0, 70, "Subsection"], Cell[60623, 1905, 1486, 30, 70, "Text"], Cell[62112, 1937, 1674, 38, 70, "Text"], Cell[63789, 1977, 290, 10, 70, "NumberedFigure", CellTags->{"Fig:2", "Ed:Change8"}], Cell[64082, 1989, 461, 11, 70, "Caption"], Cell[64546, 2002, 772, 21, 70, "Text"], Cell[65321, 2025, 290, 10, 70, "NumberedFigure", CellTags->{"Fig:3", "Ed:Change9"}], Cell[65614, 2037, 824, 12, 70, "Caption"], Cell[66441, 2051, 346, 6, 70, "Text"], Cell[66790, 2059, 772, 11, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[67599, 2075, 75, 0, 70, "Subsection"], Cell[67677, 2077, 1206, 40, 70, "Text", CellTags->"Ed:Change10"], Cell[68886, 2119, 131, 3, 70, "NumberedEquation", CellTags->"Eq:4"], Cell[69020, 2124, 1749, 50, 70, "Text", CellTags->{"Ed:Change11", "Ed:Change12"}] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[70818, 2180, 97, 5, 41, "Section", CellTags->"Sect:4"], Cell[70918, 2187, 477, 7, 70, "Text"], Cell[71398, 2196, 234, 4, 70, "Text"], Cell[CellGroupData[{ Cell[71657, 2204, 65, 0, 70, "Subsection"], Cell[71725, 2206, 407, 12, 70, "Text"], Cell[CellGroupData[{ Cell[72157, 2222, 50, 0, 70, "Subsubsection"], Cell[72210, 2224, 1017, 15, 70, "Text"], Cell[73230, 2241, 1287, 35, 70, "Text", CellTags->"Ed:Change13"], Cell[74520, 2278, 1037, 15, 70, "Text"], Cell[75560, 2295, 1093, 15, 70, "Text"], Cell[76656, 2312, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:4", "Ed:Change14"}], Cell[76950, 2324, 94, 2, 70, "Caption"], Cell[77047, 2328, 793, 17, 70, "Text"], Cell[77843, 2347, 765, 21, 70, "Text"], Cell[78611, 2370, 1397, 40, 70, "Text", CellTags->"Ed:Change15"], Cell[80011, 2412, 691, 19, 70, "Text"], Cell[80705, 2433, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:5", "Ed:Change16"}], Cell[80999, 2445, 149, 3, 70, "Caption"] }, Closed]], Cell[CellGroupData[{ Cell[81185, 2453, 46, 0, 70, "Subsubsection"], Cell[81234, 2455, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:6", "Ed:Change17"}], Cell[81528, 2467, 161, 3, 70, "Caption"], Cell[81692, 2472, 415, 13, 70, "Text"], Cell[82110, 2487, 857, 17, 70, "Text"], Cell[82970, 2506, 699, 31, 70, "Text", CellTags->{"Ed:Change18", "Ed:Change19"}], Cell[83672, 2539, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:7", "Ed:Change20"}], Cell[83966, 2551, 92, 2, 70, "Caption"], Cell[84061, 2555, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:8", "Ed:Change21"}], Cell[84355, 2567, 147, 3, 70, "Caption"], Cell[84505, 2572, 622, 9, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[85176, 2587, 65, 0, 70, "Subsection"], Cell[85244, 2589, 529, 8, 70, "Text"], Cell[CellGroupData[{ Cell[85798, 2601, 50, 0, 70, "Subsubsection"], Cell[85851, 2603, 1894, 49, 70, "Text", CellTags->{"Ed:Change22", "Ed:Change24"}], Cell[87748, 2654, 291, 10, 70, "NumberedFigure", CellTags->{"Fig:9", "Ed:Change23"}], Cell[88042, 2666, 296, 5, 70, "Caption"], Cell[88341, 2673, 1467, 25, 70, "Text"], Cell[89811, 2700, 478, 12, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[90326, 2717, 46, 0, 70, "Subsubsection"], Cell[90375, 2719, 580, 23, 70, "Text"], Cell[90958, 2744, 293, 10, 70, "NumberedFigure", CellTags->{"Fig:10", "Ed:Change25"}], Cell[91254, 2756, 417, 13, 70, "Caption"], Cell[91674, 2771, 582, 9, 70, "Text"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[92305, 2786, 38, 0, 70, "Subsection"], Cell[92346, 2788, 594, 19, 70, "Text"], Cell[CellGroupData[{ Cell[92965, 2811, 53, 0, 70, "Subsubsection"], Cell[93021, 2813, 1233, 30, 70, "Text", CellTags->"Ed:Change26"], Cell[94257, 2845, 1010, 28, 70, "Text"], Cell[95270, 2875, 293, 10, 70, "NumberedFigure", CellTags->{"Fig:11", "Ed:Change27"}], Cell[95566, 2887, 425, 7, 70, "Caption"] }, Closed]], Cell[CellGroupData[{ Cell[96028, 2899, 50, 0, 70, "Subsubsection"], Cell[96081, 2901, 293, 10, 70, "NumberedFigure", CellTags->{"Fig:12", "Ed:Change28"}], Cell[96377, 2913, 271, 5, 70, "Caption"], Cell[96651, 2920, 2620, 90, 70, "Text"], Cell[99274, 3012, 972, 21, 70, "Text"], Cell[100249, 3035, 849, 12, 70, "Text"] }, Closed]] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[101159, 3054, 75, 4, 41, "Section"], Cell[101237, 3060, 1140, 23, 70, "Text"], Cell[102380, 3085, 435, 7, 70, "Text"], Cell[102818, 3094, 497, 10, 70, "Text"], Cell[103318, 3106, 658, 13, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[104013, 3124, 35, 0, 41, "Section"], Cell[104051, 3126, 122, 3, 70, "Text"] }, Closed]], Cell[CellGroupData[{ Cell[104210, 3134, 56, 1, 70, "Section", CellTags->"Sect:Appendix"], Cell[104269, 3137, 129, 3, 70, "Text"], Cell[CellGroupData[{ Cell[104423, 3144, 59, 0, 70, "Subsection"], Cell[104485, 3146, 1314, 47, 70, "Text"], Cell[105802, 3195, 8703, 210, 70, "NumberedEquation"], Cell[114508, 3407, 1804, 63, 70, "Text", CellTags->"Ed:Change29"], Cell[116315, 3472, 2114, 59, 70, "NumberedEquation", CellTags->"Eq:6"], Cell[118432, 3533, 106, 3, 70, "Text"], Cell[118541, 3538, 805, 23, 70, "NumberedEquation", CellTags->"Ed:Change30"], Cell[119349, 3563, 663, 21, 70, "Text", CellTags->"Ed:Change31"], Cell[120015, 3586, 1981, 52, 70, "NumberedEquation", CellTags->"Ed:Change32"] }, Closed]], Cell[CellGroupData[{ Cell[122033, 3643, 83, 1, 70, "Subsection"], Cell[122119, 3646, 2578, 83, 70, "Text", CellTags->"Ed:Change33"], Cell[124700, 3731, 1571, 58, 70, "Text", CellTags->"Ed:Change34"], Cell[126274, 3791, 1916, 47, 70, "NumberedEquation", CellTags->"Ed:Change35"], Cell[128193, 3840, 433, 14, 70, "Text"], Cell[128629, 3856, 1199, 26, 70, "Text"], Cell[129831, 3884, 1895, 46, 70, "Text", CellTags->"Ed:Change36"], Cell[131729, 3932, 863, 27, 70, "Text"], Cell[132595, 3961, 3879, 117, 70, "Text", CellTags->"Ed:Change37"] }, Closed]] }, Closed]], Cell[CellGroupData[{ Cell[136523, 4084, 29, 0, 70, "Section"], Cell[136555, 4086, 210, 6, 70, "Reference", CellTags->"Ref:Brodatz1996"], Cell[136768, 4094, 201, 6, 70, "Reference", CellTags->"Ref:Kohonen1984"], Cell[136972, 4102, 278, 10, 70, "Reference", CellTags->"Ref:LindeBuzoGray1980"], Cell[137253, 4114, 486, 19, 70, "Reference", CellTags->"Ref:Luttrell1994a"], Cell[137742, 4135, 569, 19, 70, "Reference", CellTags->"Ref:Luttrell1994b"], Cell[138314, 4156, 528, 17, 70, "Reference", CellTags->"Ref:Luttrell1997a"], Cell[138845, 4175, 495, 19, 70, "Reference", CellTags->"Ref:Luttrell1997b"], Cell[139343, 4196, 575, 19, 70, "Reference", CellTags->"Ref:Luttrell1997c"], Cell[139921, 4217, 276, 10, 70, "Reference", CellTags->"Ref:Swindale1996"], Cell[140200, 4229, 296, 10, 70, "Reference", CellTags->"Ref:Webber1994"] }, Closed]] }, Open ]] } ] *) (******************************************************************* End of Mathematica Notebook file. *******************************************************************)