{
 "metadata": {
  "name": "",
  "signature": "sha256:1a2c284881d0d090a8144448ddffa2622095c15049758d95804689d0f0432b74"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from vsm import *\n",
      "from vsm.extensions.corpusbuilders import toy_corpus\n",
      "\n",
      "\n",
      "plain_corpus = \"\"\"\n",
      "His theology challenged the Pope of the Roman Catholic Church by\n",
      "teaching that the Bible is the only source of divinely revealed\n",
      "knowledge.\n",
      "\n",
      "Augustine is held in the Catholic Church to be the model teacher.\n",
      "\n",
      "Augustine was recognized as a Doctor of the Church by Pope Boniface\n",
      "VIII.\n",
      "\n",
      "Roman Catholic theology stated that faith alone cannot justify man.\n",
      "\n",
      "In the Catholic Church the Pope is regarded as the successor of Saint\n",
      "Peter.\n",
      "\n",
      "Alonzo Church was an American mathematician and logician who made\n",
      "major contributions to mathematical logic and the foundations of\n",
      "theoretical computer science.\n",
      "\n",
      "The lambda calculus was introduced by mathematician Alonzo Church as\n",
      "an investigation into the foundations of mathematics.\n",
      "\n",
      "The Church Turing thesis states that a function is algorithmically\n",
      "computable if and only if it is computable by a Turing machine.\n",
      "\n",
      "Mathematical logic has close connections to the foundations of\n",
      "mathematics, theoretical computer science.\n",
      "\n",
      "A Turing machine can be adapted to simulate the logic of any computer\n",
      "algorithm.\n",
      "\"\"\"\n",
      "\n",
      "metadata = ['Ecclesiastical ' + str(i) for i in xrange(1, 6)]\n",
      "metadata += ['Logic ' + str(i) for i in xrange(1, 6)]\n",
      "\n",
      "c = toy_corpus(plain_corpus, nltk_stop=True, metadata=metadata)\n",
      "\n",
      "tf_m = TfMulti(c, 'document')\n",
      "tf_m.train(2)\n",
      "\n",
      "tfidf_m = TfIdf(tf_m.matrix, 'document')\n",
      "tfidf_m.train()\n",
      "\n",
      "lsa_m = Lsa(tfidf_m.matrix, 'document')\n",
      "lsa_m.train(k_factors=3)\n",
      "\n",
      "tf_v = TfViewer(c, tf_m)\n",
      "tfidf_v = TfIdfViewer(c, tfidf_m)\n",
      "lsa_v = LsaViewer(c, lsa_m)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Mapping\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "/usr/lib/pymodules/python2.7/numpy/ctypeslib.py:411: RuntimeWarning: Item size computed from the PEP 3118 buffer format string does not match the actual item size.\n",
        "  return array(obj, copy=False)\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Reducing\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tf_v.dist_word_word('logic', print_len=24)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"4\">Words: logic</th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                    </th><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                    </th></tr><tr><td>logic               </td><td>0.00000   </td><td>close               </td><td>0.95532   </td></tr><tr><td>computer            </td><td>0.00000   </td><td>adapted             </td><td>0.95532   </td></tr><tr><td>theoretical         </td><td>0.61548   </td><td>algorithm           </td><td>0.95532   </td></tr><tr><td>science             </td><td>0.61548   </td><td>american            </td><td>0.95532   </td></tr><tr><td>mathematical        </td><td>0.61548   </td><td>mathematician       </td><td>1.15026   </td></tr><tr><td>foundations         </td><td>0.84107   </td><td>alonzo              </td><td>1.15026   </td></tr><tr><td>simulate            </td><td>0.95532   </td><td>machine             </td><td>1.15026   </td></tr><tr><td>major               </td><td>0.95532   </td><td>mathematics         </td><td>1.15026   </td></tr><tr><td>made                </td><td>0.95532   </td><td>turing              </td><td>1.30964   </td></tr><tr><td>logician            </td><td>0.95532   </td><td>church              </td><td>1.35081   </td></tr><tr><td>contributions       </td><td>0.95532   </td><td>pope                </td><td>1.57080   </td></tr><tr><td>connections         </td><td>0.95532   </td><td>recognized          </td><td>1.57080   </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "LabeledColumn([('logic', 0.0), ('computer', 0.0),\n",
        "       ('theoretical', 0.6154797086703874),\n",
        "       ('science', 0.6154797086703874),\n",
        "       ('mathematical', 0.6154797086703874),\n",
        "       ('foundations', 0.84106867056793), ('simulate', 0.9553166181245092),\n",
        "       ('major', 0.9553166181245092), ('made', 0.9553166181245092),\n",
        "       ('logician', 0.9553166181245092),\n",
        "       ('contributions', 0.9553166181245092),\n",
        "       ('connections', 0.9553166181245092), ('close', 0.9553166181245092),\n",
        "       ('adapted', 0.9553166181245092), ('algorithm', 0.9553166181245092),\n",
        "       ('american', 0.9553166181245092),\n",
        "       ('mathematician', 1.1502619915109316),\n",
        "       ('alonzo', 1.1502619915109316), ('machine', 1.1502619915109316),\n",
        "       ('mathematics', 1.1502619915109316), ('turing', 1.3096389158918722),\n",
        "       ('church', 1.3508083493994372), ('pope', 1.5707963267948966),\n",
        "       ('recognized', 1.5707963267948966),\n",
        "       ('regarded', 1.5707963267948966), ('revealed', 1.5707963267948966),\n",
        "       ('roman', 1.5707963267948966), ('saint', 1.5707963267948966),\n",
        "       ('algorithmically', 1.5707963267948966),\n",
        "       ('source', 1.5707963267948966), ('stated', 1.5707963267948966),\n",
        "       ('states', 1.5707963267948966), ('successor', 1.5707963267948966),\n",
        "       ('teacher', 1.5707963267948966), ('teaching', 1.5707963267948966),\n",
        "       ('theology', 1.5707963267948966), ('thesis', 1.5707963267948966),\n",
        "       ('alone', 1.5707963267948966), ('peter', 1.5707963267948966),\n",
        "       ('computable', 1.5707963267948966), ('man', 1.5707963267948966),\n",
        "       ('challenged', 1.5707963267948966),\n",
        "       ('catholic', 1.5707963267948966), ('calculus', 1.5707963267948966),\n",
        "       ('divinely', 1.5707963267948966), ('doctor', 1.5707963267948966),\n",
        "       ('faith', 1.5707963267948966), ('boniface', 1.5707963267948966),\n",
        "       ('model', 1.5707963267948966), ('function', 1.5707963267948966),\n",
        "       ('introduced', 1.5707963267948966),\n",
        "       ('investigation', 1.5707963267948966),\n",
        "       ('justify', 1.5707963267948966), ('knowledge', 1.5707963267948966),\n",
        "       ('lambda', 1.5707963267948966), ('bible', 1.5707963267948966),\n",
        "       ('augustine', 1.5707963267948966), ('held', 1.5707963267948966),\n",
        "       ('viii', 1.5707963267948966)], \n",
        "      dtype=[('word', '|S15'), ('value', '<f8')])"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tfidf_v.dist_word_word('logic', print_len=24)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"4\">Words: logic</th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                    </th><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                    </th></tr><tr><td>logic               </td><td>0.00000   </td><td>close               </td><td>0.95532   </td></tr><tr><td>computer            </td><td>0.00000   </td><td>adapted             </td><td>0.95532   </td></tr><tr><td>theoretical         </td><td>0.61548   </td><td>algorithm           </td><td>0.95532   </td></tr><tr><td>science             </td><td>0.61548   </td><td>american            </td><td>0.95532   </td></tr><tr><td>mathematical        </td><td>0.61548   </td><td>mathematician       </td><td>1.15026   </td></tr><tr><td>foundations         </td><td>0.84107   </td><td>alonzo              </td><td>1.15026   </td></tr><tr><td>simulate            </td><td>0.95532   </td><td>machine             </td><td>1.15026   </td></tr><tr><td>major               </td><td>0.95532   </td><td>mathematics         </td><td>1.15026   </td></tr><tr><td>made                </td><td>0.95532   </td><td>turing              </td><td>1.30964   </td></tr><tr><td>logician            </td><td>0.95532   </td><td>church              </td><td>1.35081   </td></tr><tr><td>contributions       </td><td>0.95532   </td><td>pope                </td><td>1.57080   </td></tr><tr><td>connections         </td><td>0.95532   </td><td>recognized          </td><td>1.57080   </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "LabeledColumn([('logic', 0.0), ('computer', 0.0),\n",
        "       ('theoretical', 0.6154797086703874),\n",
        "       ('science', 0.6154797086703874),\n",
        "       ('mathematical', 0.6154797086703874),\n",
        "       ('foundations', 0.8410686705679303),\n",
        "       ('simulate', 0.9553166181245094), ('major', 0.9553166181245094),\n",
        "       ('made', 0.9553166181245094), ('logician', 0.9553166181245094),\n",
        "       ('contributions', 0.9553166181245094),\n",
        "       ('connections', 0.9553166181245094), ('close', 0.9553166181245094),\n",
        "       ('adapted', 0.9553166181245094), ('algorithm', 0.9553166181245094),\n",
        "       ('american', 0.9553166181245094),\n",
        "       ('mathematician', 1.1502619915109316),\n",
        "       ('alonzo', 1.1502619915109316), ('machine', 1.1502619915109316),\n",
        "       ('mathematics', 1.1502619915109316), ('turing', 1.3096389158918724),\n",
        "       ('church', 1.3508083493994372), ('pope', 1.5707963267948966),\n",
        "       ('recognized', 1.5707963267948966),\n",
        "       ('regarded', 1.5707963267948966), ('revealed', 1.5707963267948966),\n",
        "       ('roman', 1.5707963267948966), ('saint', 1.5707963267948966),\n",
        "       ('algorithmically', 1.5707963267948966),\n",
        "       ('source', 1.5707963267948966), ('stated', 1.5707963267948966),\n",
        "       ('states', 1.5707963267948966), ('successor', 1.5707963267948966),\n",
        "       ('teacher', 1.5707963267948966), ('teaching', 1.5707963267948966),\n",
        "       ('theology', 1.5707963267948966), ('thesis', 1.5707963267948966),\n",
        "       ('alone', 1.5707963267948966), ('peter', 1.5707963267948966),\n",
        "       ('computable', 1.5707963267948966), ('man', 1.5707963267948966),\n",
        "       ('challenged', 1.5707963267948966),\n",
        "       ('catholic', 1.5707963267948966), ('calculus', 1.5707963267948966),\n",
        "       ('divinely', 1.5707963267948966), ('doctor', 1.5707963267948966),\n",
        "       ('faith', 1.5707963267948966), ('boniface', 1.5707963267948966),\n",
        "       ('model', 1.5707963267948966), ('function', 1.5707963267948966),\n",
        "       ('introduced', 1.5707963267948966),\n",
        "       ('investigation', 1.5707963267948966),\n",
        "       ('justify', 1.5707963267948966), ('knowledge', 1.5707963267948966),\n",
        "       ('lambda', 1.5707963267948966), ('bible', 1.5707963267948966),\n",
        "       ('augustine', 1.5707963267948966), ('held', 1.5707963267948966),\n",
        "       ('viii', 1.5707963267948966)], \n",
        "      dtype=[('word', '|S15'), ('value', '<f8')])"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lsa_v.dist_word_word('logic', print_len=24)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"4\">Words: logic</th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                    </th><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                    </th></tr><tr><td>logic               </td><td>0.00000   </td><td>contributions       </td><td>0.16133   </td></tr><tr><td>computer            </td><td>0.00000   </td><td>major               </td><td>0.16133   </td></tr><tr><td>connections         </td><td>0.14615   </td><td>alonzo              </td><td>0.16665   </td></tr><tr><td>close               </td><td>0.14615   </td><td>mathematician       </td><td>0.16665   </td></tr><tr><td>theoretical         </td><td>0.15619   </td><td>calculus            </td><td>0.18072   </td></tr><tr><td>science             </td><td>0.15619   </td><td>introduced          </td><td>0.18072   </td></tr><tr><td>mathematical        </td><td>0.15619   </td><td>lambda              </td><td>0.18072   </td></tr><tr><td>mathematics         </td><td>0.16088   </td><td>investigation       </td><td>0.18072   </td></tr><tr><td>foundations         </td><td>0.16111   </td><td>church              </td><td>0.85232   </td></tr><tr><td>made                </td><td>0.16133   </td><td>simulate            </td><td>0.94910   </td></tr><tr><td>american            </td><td>0.16133   </td><td>adapted             </td><td>0.94910   </td></tr><tr><td>logician            </td><td>0.16133   </td><td>algorithm           </td><td>0.94910   </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "LabeledColumn([('logic', 2.1073424255447017e-08),\n",
        "       ('computer', 2.1073424255447017e-08),\n",
        "       ('connections', 0.1461497327030728), ('close', 0.1461497327030728),\n",
        "       ('theoretical', 0.1561931875925689),\n",
        "       ('science', 0.1561931875925689),\n",
        "       ('mathematical', 0.1561931875925689),\n",
        "       ('mathematics', 0.16088388222038064),\n",
        "       ('foundations', 0.16111028907696195), ('made', 0.16132699126936187),\n",
        "       ('american', 0.16132699126936187),\n",
        "       ('logician', 0.16132699126936187),\n",
        "       ('contributions', 0.16132699126936187),\n",
        "       ('major', 0.16132699126936187), ('alonzo', 0.166652610177689),\n",
        "       ('mathematician', 0.166652610177689),\n",
        "       ('calculus', 0.18071500661099674),\n",
        "       ('introduced', 0.18071500661099674),\n",
        "       ('lambda', 0.18071500661099674),\n",
        "       ('investigation', 0.18071500661099674),\n",
        "       ('church', 0.8523211636647396), ('simulate', 0.9491017050433752),\n",
        "       ('adapted', 0.9491017050433752), ('algorithm', 0.9491017050433752),\n",
        "       ('machine', 1.353933874937494), ('turing', 1.3993762078367922),\n",
        "       ('thesis', 1.455329082734973), ('states', 1.455329082734973),\n",
        "       ('algorithmically', 1.455329082734973),\n",
        "       ('computable', 1.455329082734973), ('function', 1.455329082734973),\n",
        "       ('held', 1.4942581924548715), ('teacher', 1.4942581924548715),\n",
        "       ('model', 1.4942581924548715), ('augustine', 1.5036434979434157),\n",
        "       ('recognized', 1.5100453465502426), ('viii', 1.5100453465502426),\n",
        "       ('boniface', 1.5100453465502426), ('doctor', 1.5100453465502426),\n",
        "       ('peter', 1.537296434294445), ('successor', 1.537296434294445),\n",
        "       ('saint', 1.537296434294445), ('regarded', 1.537296434294445),\n",
        "       ('pope', 1.569538765830574), ('catholic', 1.5741457973418245),\n",
        "       ('challenged', 1.5786643468083752), ('bible', 1.5786643468083752),\n",
        "       ('source', 1.5786643468083752), ('knowledge', 1.5786643468083752),\n",
        "       ('teaching', 1.5786643468083752), ('revealed', 1.5786643468083752),\n",
        "       ('divinely', 1.5786643468083752), ('roman', 1.5807179416878123),\n",
        "       ('theology', 1.5807179416878123), ('alone', 1.5857499354125162),\n",
        "       ('stated', 1.5857499354125162), ('justify', 1.5857499354125162),\n",
        "       ('man', 1.5857499354125162), ('faith', 1.5857499354125162)], \n",
        "      dtype=[('word', '|S15'), ('value', '<f8')])"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tf_v.dist_doc_doc('Ecclesiastical 4')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"2\">Documents: </th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Document                          </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                          </th></tr><tr><td>Ecclesiastical 4     </td><td>0.00000   </td></tr><tr><td>Ecclesiastical 1     </td><td>1.25961   </td></tr><tr><td>Ecclesiastical 2     </td><td>1.42595   </td></tr><tr><td>Ecclesiastical 5     </td><td>1.43676   </td></tr><tr><td>Ecclesiastical 3     </td><td>1.57080   </td></tr><tr><td>Logic 1              </td><td>1.57080   </td></tr><tr><td>Logic 2              </td><td>1.57080   </td></tr><tr><td>Logic 3              </td><td>1.57080   </td></tr><tr><td>Logic 4              </td><td>1.57080   </td></tr><tr><td>Logic 5              </td><td>1.57080   </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "LabeledColumn([('Ecclesiastical 4', 1.4901161193847656e-08),\n",
        "       ('Ecclesiastical 1', 1.2596120825173864),\n",
        "       ('Ecclesiastical 2', 1.4259528297963369),\n",
        "       ('Ecclesiastical 5', 1.4367647653836777),\n",
        "       ('Ecclesiastical 3', 1.5707963267948966),\n",
        "       ('Logic 1', 1.5707963267948966), ('Logic 2', 1.5707963267948966),\n",
        "       ('Logic 3', 1.5707963267948966), ('Logic 4', 1.5707963267948966),\n",
        "       ('Logic 5', 1.5707963267948966)], \n",
        "      dtype=[('doc', '|S16'), ('value', '<f8')])"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tfidf_v.dist_doc_doc('Ecclesiastical 4')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"2\">Documents: </th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Document                          </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                          </th></tr><tr><td>Ecclesiastical 4     </td><td>0.00000   </td></tr><tr><td>Ecclesiastical 1     </td><td>1.41228   </td></tr><tr><td>Ecclesiastical 2     </td><td>1.53742   </td></tr><tr><td>Ecclesiastical 5     </td><td>1.54051   </td></tr><tr><td>Ecclesiastical 3     </td><td>1.57080   </td></tr><tr><td>Logic 1              </td><td>1.57080   </td></tr><tr><td>Logic 2              </td><td>1.57080   </td></tr><tr><td>Logic 3              </td><td>1.57080   </td></tr><tr><td>Logic 4              </td><td>1.57080   </td></tr><tr><td>Logic 5              </td><td>1.57080   </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "LabeledColumn([('Ecclesiastical 4', 0.0), ('Ecclesiastical 1', 1.4122756403782417),\n",
        "       ('Ecclesiastical 2', 1.5374225855153376),\n",
        "       ('Ecclesiastical 5', 1.5405050906164508),\n",
        "       ('Ecclesiastical 3', 1.5707963267948966),\n",
        "       ('Logic 1', 1.5707963267948966), ('Logic 2', 1.5707963267948966),\n",
        "       ('Logic 3', 1.5707963267948966), ('Logic 4', 1.5707963267948966),\n",
        "       ('Logic 5', 1.5707963267948966)], \n",
        "      dtype=[('doc', '|S16'), ('value', '<f8')])"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tfidf_v.dist_word_doc('computer')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"2\">Words: </th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Document                          </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                          </th></tr><tr><td>Logic 5              </td><td>1.32242   </td></tr><tr><td>Logic 4              </td><td>1.32915   </td></tr><tr><td>Logic 1              </td><td>1.38815   </td></tr><tr><td>Ecclesiastical 1     </td><td>1.57080   </td></tr><tr><td>Ecclesiastical 2     </td><td>1.57080   </td></tr><tr><td>Ecclesiastical 3     </td><td>1.57080   </td></tr><tr><td>Ecclesiastical 4     </td><td>1.57080   </td></tr><tr><td>Ecclesiastical 5     </td><td>1.57080   </td></tr><tr><td>Logic 2              </td><td>1.57080   </td></tr><tr><td>Logic 3              </td><td>1.57080   </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "LabeledColumn([('Logic 5', 1.3224154697632784), ('Logic 4', 1.3291532220319409),\n",
        "       ('Logic 1', 1.388146251745384),\n",
        "       ('Ecclesiastical 1', 1.5707963267948966),\n",
        "       ('Ecclesiastical 2', 1.5707963267948966),\n",
        "       ('Ecclesiastical 3', 1.5707963267948966),\n",
        "       ('Ecclesiastical 4', 1.5707963267948966),\n",
        "       ('Ecclesiastical 5', 1.5707963267948966),\n",
        "       ('Logic 2', 1.5707963267948966), ('Logic 3', 1.5707963267948966)], \n",
        "      dtype=[('doc', '|S16'), ('value', '<f8')])"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lsa_v.dist_word_doc('church')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"2\">Words: </th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Document                          </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                          </th></tr><tr><td>Ecclesiastical 2     </td><td>0.73633   </td></tr><tr><td>Ecclesiastical 3     </td><td>0.75591   </td></tr><tr><td>Ecclesiastical 5     </td><td>0.78818   </td></tr><tr><td>Ecclesiastical 1     </td><td>0.83776   </td></tr><tr><td>Logic 5              </td><td>0.84555   </td></tr><tr><td>Ecclesiastical 4     </td><td>0.84617   </td></tr><tr><td>Logic 4              </td><td>0.94199   </td></tr><tr><td>Logic 2              </td><td>0.94433   </td></tr><tr><td>Logic 1              </td><td>0.94556   </td></tr><tr><td>Logic 3              </td><td>1.11314   </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "LabeledColumn([('Ecclesiastical 2', 0.7363264837344443),\n",
        "       ('Ecclesiastical 3', 0.7559069891816871),\n",
        "       ('Ecclesiastical 5', 0.7881792424138612),\n",
        "       ('Ecclesiastical 1', 0.8377568378998803),\n",
        "       ('Logic 5', 0.8455464728606694),\n",
        "       ('Ecclesiastical 4', 0.8461724249009318),\n",
        "       ('Logic 4', 0.9419921003724298), ('Logic 2', 0.9443332777787585),\n",
        "       ('Logic 1', 0.9455622593137681), ('Logic 3', 1.1131440244046271)], \n",
        "      dtype=[('doc', '|S16'), ('value', '<f8')])"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lsa_v.dist_doc_doc('Ecclesiastical 4')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"2\">Documents: </th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Document                          </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                          </th></tr><tr><td>Ecclesiastical 4     </td><td>0.00000   </td></tr><tr><td>Ecclesiastical 1     </td><td>0.00843   </td></tr><tr><td>Ecclesiastical 5     </td><td>0.05804   </td></tr><tr><td>Ecclesiastical 3     </td><td>0.09036   </td></tr><tr><td>Ecclesiastical 2     </td><td>0.10991   </td></tr><tr><td>Logic 2              </td><td>1.56321   </td></tr><tr><td>Logic 3              </td><td>1.57501   </td></tr><tr><td>Logic 1              </td><td>1.57808   </td></tr><tr><td>Logic 4              </td><td>1.58446   </td></tr><tr><td>Logic 5              </td><td>1.58765   </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "LabeledColumn([('Ecclesiastical 4', 0.0),\n",
        "       ('Ecclesiastical 1', 0.008426996338275176),\n",
        "       ('Ecclesiastical 5', 0.05803666882089707),\n",
        "       ('Ecclesiastical 3', 0.09036042875240273),\n",
        "       ('Ecclesiastical 2', 0.10990663361013094),\n",
        "       ('Logic 2', 1.563206166779967), ('Logic 3', 1.5750098468112752),\n",
        "       ('Logic 1', 1.5780801982346484), ('Logic 4', 1.58446244677414),\n",
        "       ('Logic 5', 1.5876493531936278)], \n",
        "      dtype=[('doc', '|S16'), ('value', '<f8')])"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tf_v.coll_freq('church')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "7"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tf_v.coll_freqs(print_len=54)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"8\">Collection Frequencies</th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Counts                    </th><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Counts                    </th><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Counts                    </th><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Counts                    </th></tr><tr><td>church              </td><td>7         </td><td>computable          </td><td>2         </td><td>calculus            </td><td>1         </td><td>states              </td><td>1         </td></tr><tr><td>catholic            </td><td>4         </td><td>augustine           </td><td>2         </td><td>american            </td><td>1         </td><td>stated              </td><td>1         </td></tr><tr><td>foundations         </td><td>3         </td><td>roman               </td><td>2         </td><td>close               </td><td>1         </td><td>source              </td><td>1         </td></tr><tr><td>turing              </td><td>3         </td><td>science             </td><td>2         </td><td>algorithmically     </td><td>1         </td><td>simulate            </td><td>1         </td></tr><tr><td>computer            </td><td>3         </td><td>alone               </td><td>1         </td><td>held                </td><td>1         </td><td>saint               </td><td>1         </td></tr><tr><td>pope                </td><td>3         </td><td>contributions       </td><td>1         </td><td>function            </td><td>1         </td><td>revealed            </td><td>1         </td></tr><tr><td>logic               </td><td>3         </td><td>algorithm           </td><td>1         </td><td>viii                </td><td>1         </td><td>regarded            </td><td>1         </td></tr><tr><td>alonzo              </td><td>2         </td><td>faith               </td><td>1         </td><td>introduced          </td><td>1         </td><td>recognized          </td><td>1         </td></tr><tr><td>mathematics         </td><td>2         </td><td>doctor              </td><td>1         </td><td>investigation       </td><td>1         </td><td>peter               </td><td>1         </td></tr><tr><td>theoretical         </td><td>2         </td><td>divinely            </td><td>1         </td><td>thesis              </td><td>1         </td><td>model               </td><td>1         </td></tr><tr><td>machine             </td><td>2         </td><td>bible               </td><td>1         </td><td>teaching            </td><td>1         </td><td>man                 </td><td>1         </td></tr><tr><td>theology            </td><td>2         </td><td>boniface            </td><td>1         </td><td>teacher             </td><td>1         </td><td>major               </td><td>1         </td></tr><tr><td>mathematical        </td><td>2         </td><td>connections         </td><td>1         </td><td>successor           </td><td>1         </td><td>made                </td><td>1         </td></tr><tr><td>mathematician       </td><td>2         </td><td>challenged          </td><td>1         </td><td style=\"border-color: #EFF2FB; background: #EEF2FB;\">                                                 </td><td style=\"border-color: #EFF2FB; background: #EEF2FB;\">                                       </td><td style=\"border-color: #EFF2FB; background: #EEF2FB;\">                                                 </td><td style=\"border-color: #EFF2FB; background: #EEF2FB;\">                                       </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "LabeledColumn([('church', 7), ('catholic', 4), ('foundations', 3), ('turing', 3),\n",
        "       ('computer', 3), ('pope', 3), ('logic', 3), ('alonzo', 2),\n",
        "       ('mathematics', 2), ('theoretical', 2), ('machine', 2),\n",
        "       ('theology', 2), ('mathematical', 2), ('mathematician', 2),\n",
        "       ('computable', 2), ('augustine', 2), ('roman', 2), ('science', 2),\n",
        "       ('alone', 1), ('contributions', 1), ('algorithm', 1), ('faith', 1),\n",
        "       ('doctor', 1), ('divinely', 1), ('bible', 1), ('boniface', 1),\n",
        "       ('connections', 1), ('challenged', 1), ('calculus', 1),\n",
        "       ('american', 1), ('close', 1), ('algorithmically', 1), ('held', 1),\n",
        "       ('function', 1), ('viii', 1), ('introduced', 1),\n",
        "       ('investigation', 1), ('thesis', 1), ('teaching', 1),\n",
        "       ('teacher', 1), ('successor', 1), ('states', 1), ('stated', 1),\n",
        "       ('source', 1), ('simulate', 1), ('saint', 1), ('revealed', 1),\n",
        "       ('regarded', 1), ('recognized', 1), ('peter', 1), ('model', 1),\n",
        "       ('man', 1), ('major', 1), ('made', 1), ('logician', 1),\n",
        "       ('lambda', 1), ('knowledge', 1), ('justify', 1), ('adapted', 1)], \n",
        "      dtype=[('word', '|S15'), ('value', '<i4')])"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dm = tf_v.dismat_word(['logic','church','catholic','pope'])\n",
      "np.around(dm, decimals=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 12,
       "text": [
        "array([[ 0.  ,  1.35,  1.57,  1.57],\n",
        "       [ 1.35,  0.  ,  0.97,  0.86],\n",
        "       [ 1.57,  0.97,  0.  ,  0.96],\n",
        "       [ 1.57,  0.86,  0.96,  0.  ]])"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dm = tfidf_v.dismat_word(['logic','church','catholic','pope'])\n",
      "np.around(dm, decimals=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 13,
       "text": [
        "array([[ 0.  ,  1.35,  1.57,  1.57],\n",
        "       [ 1.35,  0.  ,  0.97,  0.86],\n",
        "       [ 1.57,  0.97,  0.  ,  0.96],\n",
        "       [ 1.57,  0.86,  0.96,  0.  ]])"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dm = lsa_v.dismat_word(['logic','church','catholic','pope'])\n",
      "np.around(dm, decimals=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 14,
       "text": [
        "array([[ 0.  ,  0.85,  1.57,  1.57],\n",
        "       [ 0.85,  0.  ,  0.87,  0.87],\n",
        "       [ 1.57,  0.87,  0.  ,  0.01],\n",
        "       [ 1.57,  0.87,  0.01,  0.  ]])"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dm = tf_v.dismat_doc(['Logic 1','Logic 2','Ecclesiastical 1','Ecclesiastical 2'])\n",
      "np.around(dm, decimals=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 15,
       "text": [
        "array([[ 0.  ,  1.21,  1.49,  1.46],\n",
        "       [ 1.21,  0.  ,  1.47,  1.43],\n",
        "       [ 1.49,  1.47,  0.  ,  1.33],\n",
        "       [ 1.46,  1.43,  1.33,  0.  ]])"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dm = tfidf_v.dismat_doc(['Logic 1','Logic 2','Ecclesiastical 1','Ecclesiastical 2'])\n",
      "np.around(dm, decimals=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 16,
       "text": [
        "array([[ 0.  ,  1.39,  1.57,  1.57],\n",
        "       [ 1.39,  0.  ,  1.57,  1.57],\n",
        "       [ 1.57,  1.57,  0.  ,  1.54],\n",
        "       [ 1.57,  1.57,  1.54,  0.  ]])"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dm = lsa_v.dismat_doc(['Logic 1','Logic 2','Ecclesiastical 1','Ecclesiastical 2'])\n",
      "np.around(dm, decimals=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 17,
       "text": [
        "array([[ 0.  ,  0.02,  1.57,  1.49],\n",
        "       [ 0.02,  0.  ,  1.56,  1.48],\n",
        "       [ 1.57,  1.56,  0.  ,  0.1 ],\n",
        "       [ 1.49,  1.48,  0.1 ,  0.  ]])"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from vsm.spatial import scipy_cdist\n",
      "lsa_v.dist_word_word('logic', print_len=24, dist_fn=scipy_cdist(metric='minkowski'))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<table style=\"margin: 0\"><tr><th style=\"text-align: center; background: #CEE3F6\" colspan                    =\"4\">Words: logic</th></tr><tr><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                    </th><th style=\"text-align: center; background: #EFF2FB; \">Word                    </th><th style=\"text-align: center; background: #EFF2FB; \">Distance                    </th></tr><tr><td>logic               </td><td>0.00000   </td><td>theoretical         </td><td>0.06325   </td></tr><tr><td>computer            </td><td>0.00000   </td><td>mathematics         </td><td>0.07048   </td></tr><tr><td>mathematician       </td><td>0.04921   </td><td>connections         </td><td>0.09843   </td></tr><tr><td>alonzo              </td><td>0.04921   </td><td>close               </td><td>0.09843   </td></tr><tr><td>foundations         </td><td>0.05166   </td><td>introduced          </td><td>0.12865   </td></tr><tr><td>american            </td><td>0.05320   </td><td>lambda              </td><td>0.12865   </td></tr><tr><td>contributions       </td><td>0.05320   </td><td>investigation       </td><td>0.12865   </td></tr><tr><td>logician            </td><td>0.05320   </td><td>calculus            </td><td>0.12865   </td></tr><tr><td>made                </td><td>0.05320   </td><td>church              </td><td>0.18155   </td></tr><tr><td>major               </td><td>0.05320   </td><td>simulate            </td><td>0.19473   </td></tr><tr><td>mathematical        </td><td>0.06325   </td><td>adapted             </td><td>0.19473   </td></tr><tr><td>science             </td><td>0.06325   </td><td>algorithm           </td><td>0.19473   </td></tr></table>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 18,
       "text": [
        "LabeledColumn([('logic', 0.0), ('computer', 0.0),\n",
        "       ('mathematician', 0.0492069763182823),\n",
        "       ('alonzo', 0.0492069763182823),\n",
        "       ('foundations', 0.05166368875941544),\n",
        "       ('american', 0.05320430492811313),\n",
        "       ('contributions', 0.05320430492811313),\n",
        "       ('logician', 0.05320430492811313), ('made', 0.05320430492811313),\n",
        "       ('major', 0.05320430492811313),\n",
        "       ('mathematical', 0.0632503959697633),\n",
        "       ('science', 0.0632503959697633),\n",
        "       ('theoretical', 0.0632503959697633),\n",
        "       ('mathematics', 0.07048230197449873),\n",
        "       ('connections', 0.09842980401516312),\n",
        "       ('close', 0.09842980401516312), ('introduced', 0.12864998450849807),\n",
        "       ('lambda', 0.12864998450849807),\n",
        "       ('investigation', 0.12864998450849807),\n",
        "       ('calculus', 0.12864998450849807), ('church', 0.18154738827057404),\n",
        "       ('simulate', 0.19473020810363867), ('adapted', 0.19473020810363867),\n",
        "       ('algorithm', 0.19473020810363867), ('held', 0.22833958546575822),\n",
        "       ('teacher', 0.22833958546575822), ('model', 0.22833958546575822),\n",
        "       ('recognized', 0.22893465574112978),\n",
        "       ('boniface', 0.22893465574112978), ('viii', 0.22893465574112978),\n",
        "       ('doctor', 0.22893465574112978), ('augustine', 0.2290393895483237),\n",
        "       ('peter', 0.23082222876942637), ('saint', 0.23082222876942637),\n",
        "       ('successor', 0.23082222876942637),\n",
        "       ('regarded', 0.23082222876942637), ('alone', 0.26236426820101866),\n",
        "       ('justify', 0.26236426820101866), ('stated', 0.26236426820101866),\n",
        "       ('faith', 0.26236426820101866), ('man', 0.26236426820101866),\n",
        "       ('pope', 0.2993972085117987), ('catholic', 0.30013287605358824),\n",
        "       ('machine', 0.3040698811395842),\n",
        "       ('algorithmically', 0.35328304712159975),\n",
        "       ('function', 0.35328304712159975), ('states', 0.35328304712159975),\n",
        "       ('thesis', 0.35328304712159975), ('theology', 0.3796994235501325),\n",
        "       ('roman', 0.3796994235501325), ('revealed', 0.38318073578287704),\n",
        "       ('knowledge', 0.38318073578287704),\n",
        "       ('teaching', 0.38318073578287704), ('bible', 0.38318073578287704),\n",
        "       ('divinely', 0.38318073578287704),\n",
        "       ('challenged', 0.38318073578287704),\n",
        "       ('source', 0.38318073578287704), ('turing', 0.47942958346148457),\n",
        "       ('computable', 0.6109097308593834)], \n",
        "      dtype=[('word', '|S15'), ('value', '<f8')])"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dm = lsa_v.dismat_doc(['Logic 1','Logic 2','Ecclesiastical 1','Ecclesiastical 2'], dist_fn=scipy_cdist(metric='cosine'))\n",
      "np.around(dm, decimals=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 19,
       "text": [
        "array([[ 0.  ,  0.  ,  1.  ,  0.92],\n",
        "       [ 0.  ,  0.  ,  0.99,  0.91],\n",
        "       [ 1.  ,  0.99,  0.  ,  0.01],\n",
        "       [ 0.92,  0.91,  0.01,  0.  ]])"
       ]
      }
     ],
     "prompt_number": 19
    }
   ],
   "metadata": {}
  }
 ]
}