from spacy_ebooks import Book
import plotly
import pandas as pd
from sentence_transformers.util import community_detection, dot_score
import sentence_transformers.util as util
import spacy_sentence_bert

import spacy

spacy.prefer_gpu()

# nlp = spacy_sentence_bert.load_model("en_stsb_roberta_large")  #  "all-MiniLM-L6-v2"

nlp = spacy.blank("en")
nlp.add_pipe("sentence_bert", config={"model_name": "all-mpnet-base-v2"})
nlp.pipe_names

['sentence_bert']

book = Book("../../resources/a-merritt_the-moon-pool_advanced.epub3")

for doc in nlp.pipe(p.text() for p in book.paragraphs()):
    print(doc, doc.vector, doc.vector.shape)
    break

I [ 1.18124895e-02  4.85588796e-02 -2.88169477e-02 -3.14895511e-02
 -1.71792111e-03  3.07480252e-04 -5.01420945e-02 -1.22614554e-03
  2.36570705e-02  3.90915535e-02 -3.22098233e-04 -9.34572890e-03
  3.34772915e-02 -8.03104881e-03  2.62643266e-02 -1.29776923e-02
  1.95034426e-02  9.35236400e-04 -5.74895814e-02  1.39178419e-02
 -1.46445101e-02  3.81998047e-02  3.26465257e-02 -6.11815322e-03
 -8.43799207e-03 -4.16335948e-02  4.99221496e-02  2.28105905e-03
  8.83174303e-04 -2.93523408e-02  8.18206295e-02 -3.02672796e-02
 -1.07187759e-02 -1.79097466e-02  2.56418730e-06  4.40120697e-02
  4.86003831e-02  9.12620034e-03 -3.87371741e-02  2.08307523e-02
  7.88209308e-03 -2.00574119e-02 -2.09908728e-02  4.55640629e-02
  3.86874913e-03  1.55576197e-02  7.32957497e-02  2.55871415e-02
 -4.75375466e-02 -5.02224155e-02 -2.69098673e-02 -1.07480017e-02
  1.19321560e-02 -3.89483757e-02  1.66534837e-02 -3.05532496e-02
  5.89303486e-03 -1.50818583e-02  1.17888199e-02 -1.32739474e-03
  5.93005912e-03  2.16858443e-02  1.56369042e-02  2.21260656e-02
  5.14838705e-03  1.13378069e-03  1.07409386e-02  2.10788250e-02
  6.24945946e-03 -2.36530807e-02  2.53051724e-02 -1.58842802e-02
 -3.82992998e-02  1.06649540e-01 -4.69332300e-02 -1.81131642e-02
 -3.94816510e-02  1.79588515e-02 -3.28306481e-02 -3.18176448e-02
 -1.93735138e-02  7.38478452e-02  3.41599062e-02  1.67910159e-02
 -1.35507323e-02  1.30229658e-02  1.84073746e-02 -4.07844782e-04
 -3.70418243e-02  4.38370965e-02 -9.90311056e-03 -1.52428476e-02
  7.87118822e-03  1.83159299e-02 -4.98853736e-02  4.63229930e-03
 -2.56460942e-02  9.45457723e-03 -2.54303310e-02  2.15147831e-03
  1.33136578e-04  1.54308202e-02  2.54046246e-02  1.54980291e-02
  8.57047085e-03  1.78176463e-02 -7.98000209e-03 -2.21727174e-02
 -5.52312145e-03  3.90486345e-02 -1.65630144e-03  1.35805504e-02
  2.16449443e-02  2.05171313e-02  4.72815149e-02 -3.10824928e-03
 -3.37069593e-02  4.24849354e-02  3.32454219e-02  2.22804975e-02
 -9.50578228e-02  1.60117373e-02 -2.94580567e-03  1.42147886e-02
  5.54166324e-02  6.64255694e-02 -1.69447232e-02  4.37570065e-02
 -3.15025300e-02  3.90573293e-02  2.14547608e-02 -2.32829172e-02
  1.74675919e-02  1.39158126e-03 -4.04391717e-03 -2.33110767e-02
 -3.18179168e-02 -1.38684018e-02 -2.18508132e-02 -3.40981744e-02
 -3.76580693e-02  1.28372908e-02 -5.28654009e-02 -5.36749065e-02
 -4.21822518e-02 -2.45998390e-02  1.82272622e-03 -9.73964110e-02
 -2.64244024e-02 -5.72370812e-02 -1.30735966e-03  6.21523112e-02
 -4.43851575e-02  3.51023749e-02 -4.82302206e-03  3.01287025e-02
  1.38484286e-02 -7.09827766e-02 -4.22320813e-02  2.90752370e-02
 -6.88352669e-03  1.85103007e-02  3.67714278e-02  1.08451014e-02
 -1.56316664e-02  4.63744923e-02  8.55027046e-03 -1.96506567e-02
 -8.38246420e-02 -5.18247345e-03  3.70031665e-03  1.87086668e-02
 -4.13836315e-02  8.65559652e-03  5.62072434e-02  1.05826519e-01
 -3.38350907e-02  1.96300703e-03  2.36353669e-02 -9.45148338e-03
 -1.34950867e-02 -3.31769721e-03  3.38449515e-02 -1.38910040e-02
  1.17460440e-03  1.59295257e-02 -8.54099449e-03  3.66765843e-03
 -1.88436564e-02  5.00974292e-03  3.08761699e-03 -4.00799839e-03
 -1.67880096e-02  5.19005321e-02  1.84378996e-02 -6.66416192e-04
  1.97340106e-03 -1.48887644e-02  3.17692384e-02  1.31120961e-02
  1.23593463e-02  1.91688631e-02  5.98079786e-02  4.04048823e-02
  1.54597331e-02 -4.05590199e-02  3.71334180e-02 -1.56006081e-05
 -1.88760576e-03  3.64210419e-02  7.30709136e-02 -2.32836735e-02
 -2.88689323e-02  2.28654295e-02  4.04572822e-02  4.69790846e-02
  9.66105051e-03  2.77408380e-02  3.01923547e-02  4.58198935e-02
 -4.97469902e-02 -1.49230985e-02  1.51738981e-02 -3.12196743e-03
 -1.24150906e-02  2.45028175e-02  9.86939576e-03  2.86383219e-02
  3.02726906e-02 -8.66573676e-03  6.81585725e-03 -5.92789007e-03
  2.81208986e-03  1.05589619e-02 -1.99644472e-02  4.65971865e-02
  3.88704948e-02 -7.19814226e-02 -8.31903219e-02 -1.03694849e-01
 -3.56161362e-03  1.82038862e-02  2.39286162e-02 -1.91431493e-02
  3.69903557e-02 -6.19188026e-02  1.72505323e-02 -3.13799903e-02
 -5.95681928e-03  2.71705911e-02 -3.81638319e-03 -5.93619533e-02
  1.32515822e-02  4.49321494e-02  9.66263376e-03 -1.98387424e-03
 -4.92050573e-02 -2.47775950e-02 -4.84641679e-02 -8.44242889e-03
  7.19182938e-02 -4.98572402e-02 -5.39836772e-02  5.00450209e-02
  2.59688180e-02  3.51871783e-03  4.57571074e-02 -1.00280717e-01
 -4.73164208e-03  1.50947471e-05 -3.54779363e-02 -4.49665710e-02
 -4.22534300e-03  1.88432410e-02  3.81765738e-02  1.03396913e-02
 -4.40810099e-02  7.57495116e-04 -4.91958559e-02  2.44154539e-02
  1.47595499e-02  5.45976916e-03 -9.90983378e-03  2.24891882e-02
  4.29822691e-02  1.71158053e-02 -2.46423706e-02 -8.09869319e-02
  2.25682072e-02 -3.41726206e-02 -2.02233512e-02 -4.63868771e-03
  5.81115671e-02 -1.27561055e-02  5.51225170e-02  1.58630013e-02
  7.34954467e-03  6.41754046e-02 -1.90057736e-02 -9.39921755e-03
  1.99836213e-02 -3.54834720e-02 -3.73099744e-02  3.16395611e-02
  9.27345641e-03 -3.80103849e-02  2.42583366e-04 -5.49288467e-02
 -4.92898189e-03  5.33188172e-02  4.74203415e-02  5.20689450e-02
 -5.03055705e-03 -1.57746151e-02  2.95899902e-02 -2.16215830e-02
 -5.02776355e-03 -4.74660955e-02 -2.51175789e-03  3.27450596e-02
  4.87629371e-03 -1.85303725e-02  1.52765103e-02  1.33343283e-02
 -5.06165549e-02  3.84851471e-02 -2.40695830e-02  2.07088571e-02
 -3.53251845e-02 -2.25227196e-02  2.03929655e-02  1.99828167e-02
  4.39691581e-02 -5.27819172e-02  3.70180458e-02 -3.97809455e-03
  2.74445042e-02  1.01147015e-02 -5.18322475e-02  9.70604364e-03
 -1.61112770e-02 -2.51816269e-02 -3.45836133e-02 -9.44256317e-03
 -6.92731217e-02 -3.97139192e-02 -4.26434092e-02  2.71762293e-02
 -4.55697887e-02  6.61580916e-03 -7.40429312e-02  1.22813582e-02
 -2.46618111e-02 -5.00722118e-02  9.94199235e-03  5.29249720e-02
 -3.75496745e-02  3.35231796e-03 -1.87599100e-02  1.08890329e-02
 -7.09292665e-02  1.67759378e-02  5.50218709e-02  1.01152109e-03
  8.65851995e-03 -2.31730985e-03  7.90733546e-02  2.76358481e-02
 -4.47780937e-02  3.52081172e-02 -1.50684714e-02  4.57180999e-02
 -3.80938612e-02  2.78872307e-02  1.77643250e-03 -1.16682872e-02
  5.50274253e-02  2.82372665e-02  4.38137613e-02 -5.13805561e-02
  9.12947580e-03 -8.37574229e-02 -4.40677293e-02  1.38852010e-02
  7.93101266e-03 -4.74128239e-02  3.87034169e-03 -4.24171276e-02
 -7.47333989e-02  6.81650545e-03  5.66665120e-02  2.76902202e-03
 -1.23228133e-02  5.11391647e-02  6.84464350e-02  5.83433956e-02
 -5.70993610e-02  3.89930904e-02 -3.25069055e-02  2.56743338e-02
 -5.65190352e-02  1.19927078e-01 -1.22698315e-03 -4.23247169e-04
 -5.25395013e-03 -1.19870286e-02  1.89163201e-02  3.63846198e-02
  2.15014014e-02 -2.96125412e-02 -3.54494690e-03  4.64193523e-02
  6.42683823e-03  3.40097398e-02  5.13208248e-02 -6.66161999e-02
  6.44842861e-03 -6.00232109e-02  1.73083898e-02 -3.89812291e-02
 -2.83748768e-02  1.40857380e-02 -7.00300047e-03 -6.37414157e-02
 -9.47010219e-02  3.36741023e-02 -3.86993308e-03 -2.84992289e-02
  8.13039020e-03  1.14217550e-02  1.47954915e-02  2.21403241e-02
 -3.38787585e-02  1.89924370e-02 -2.23320741e-02 -4.98069301e-02
  6.29462628e-03  5.33417650e-02 -8.77633505e-03 -5.06770201e-02
  4.39690240e-03 -1.54939508e-02  6.60374612e-02 -3.83634232e-02
  2.22920487e-03  4.42314474e-03  2.10568830e-02  4.01097871e-02
 -1.18875317e-01  2.82042641e-02 -5.81374252e-03  1.13085918e-02
 -1.16827367e-02  1.66747216e-02  4.12560999e-02 -1.95528120e-02
  6.53721169e-02 -2.90956739e-02  3.97283211e-02  2.80018598e-02
  6.59763142e-02  4.43391204e-02  3.39064337e-02 -1.96465626e-02
  3.19738127e-02 -3.81957889e-02 -8.71816557e-03 -8.85401294e-03
 -9.26859304e-03 -4.09288928e-02 -7.43510947e-02 -2.20598150e-02
 -8.11495446e-03 -4.31425609e-02  4.42693122e-02  5.18193990e-02
 -1.85666177e-02 -9.68685094e-03 -4.61194254e-02  2.36740261e-02
  1.48089994e-02 -7.50765437e-03 -3.12018264e-02  8.31810478e-03
  5.63032664e-02 -1.34612927e-02  1.31807635e-02 -1.19193085e-02
 -1.22228926e-02 -1.27531951e-02  3.08560692e-02 -1.12287828e-03
 -1.73919648e-03 -7.68426852e-03  3.07372101e-02 -6.02084305e-03
 -1.83288697e-02 -4.58513536e-02 -7.70268869e-03  2.22004727e-02
  1.41672426e-04 -7.16307387e-03  1.88588332e-02  5.25215007e-02
 -2.76163593e-02  9.08911135e-03  3.41331889e-03 -3.16086672e-02
  3.93384360e-02 -8.60075355e-02  1.50887994e-02  3.14201787e-02
 -1.53296413e-02 -2.97858752e-02 -3.09581810e-04  1.99964810e-02
 -3.19697522e-02 -5.58578558e-02 -9.01169628e-02 -4.52428535e-02
  8.07022490e-03  5.98727399e-03  3.65223400e-02  3.01364474e-02
  8.17405339e-03  2.56520510e-02  3.80223617e-03 -6.62919208e-02
 -1.16574625e-02  4.28041779e-02 -2.76401127e-03 -8.22357312e-02
  7.59892687e-02  5.25143789e-03 -6.46932647e-02 -1.29461866e-02
 -2.58833598e-02  3.08163688e-02 -4.39462392e-03  1.12318859e-01
  5.59213720e-02  9.34286683e-04  6.50774240e-02 -6.64903745e-02
  7.10922480e-02  4.99024466e-02 -2.22271848e-02  2.59386264e-02
 -4.97431215e-03 -2.08405238e-02 -5.41748432e-03  9.61956661e-03
 -7.33594820e-02 -3.49772573e-02 -8.62023756e-02 -1.05993055e-32
 -2.28481051e-02  1.33642601e-02 -7.07295761e-02  3.14956382e-02
 -4.83647324e-02 -4.06586230e-02 -2.37930920e-02  1.07277557e-02
  2.38286890e-02 -1.03126513e-03 -6.52374625e-02 -1.59665924e-02
  4.69593853e-02 -2.37258729e-02  1.40010472e-03 -1.94089692e-02
  5.71153313e-02  5.66725358e-02 -6.78146482e-02 -2.59530544e-02
  1.50697371e-02  3.87978996e-03  7.07806498e-02  2.45347358e-02
 -2.19923873e-02 -9.29541653e-04 -2.13970132e-02 -1.26458816e-02
  3.60783353e-03  3.50924544e-02 -3.78546608e-03  3.62029150e-02
 -1.74686089e-02  2.76157502e-02  2.05156133e-02  7.99749941e-02
 -3.14014181e-02 -2.70666573e-02 -2.14732811e-02 -1.57126319e-02
 -2.47418545e-02 -6.55945987e-02  9.22863632e-02 -3.81656811e-02
  1.03917046e-04 -1.43378926e-02  6.85506314e-03 -1.10813649e-02
 -1.45469056e-02  1.76769011e-02 -4.93514985e-02 -5.24119521e-03
  3.59493168e-03 -8.70734174e-03 -2.84681376e-02  4.43634167e-02
  2.06427630e-02  1.34209264e-02 -3.47532332e-02 -1.18151568e-02
 -4.80284467e-02 -2.96585131e-02 -7.05845132e-02  5.26801012e-02
  3.95793915e-02  3.77141461e-02  5.32438047e-02  5.11728600e-02
 -1.20905293e-02  2.00982820e-02 -2.84120347e-02 -5.72140969e-04
 -2.11974867e-02  2.15334818e-02 -8.50970000e-02 -3.74290682e-02
  1.65669806e-02  3.50883529e-02  4.35262360e-02 -2.07609888e-02
 -3.31237377e-03 -3.78281856e-03 -5.76437972e-02 -4.42247503e-02
  2.89639179e-03  6.96997973e-04 -4.01202496e-03  1.51011851e-02
 -1.96068063e-02 -4.14261632e-02  3.92477959e-03  1.38858901e-02
 -2.95831310e-03 -2.11048368e-02 -6.87715858e-02 -4.66591977e-02
 -1.15619868e-03  4.01152708e-02  9.02528060e-04 -7.83502124e-03
  5.08552901e-02  8.86452571e-02  3.37742977e-02  6.61473349e-02
  1.49279386e-02  1.05862608e-02  3.47109661e-02  1.46355322e-02
 -3.59649956e-02 -4.82601039e-02  6.31571235e-03  5.13339136e-03
  6.14324175e-02  2.41008438e-02  3.97571027e-02  5.81262633e-02
 -1.94999226e-03 -5.47600500e-02 -8.07359070e-02  4.22159135e-02
 -2.12794878e-02 -2.10346319e-02 -7.89809301e-02 -3.07654366e-02
 -7.71509856e-03  1.95554011e-02 -3.76308374e-02 -2.80799642e-02
 -7.97554478e-03 -7.80153228e-03 -6.39149081e-03 -4.43143584e-03
  3.36637726e-07 -1.51652368e-02 -2.97388118e-02  2.53911316e-03
 -9.21839848e-03 -1.82452779e-02  9.69864428e-03 -8.96314159e-03
  2.10281983e-02 -7.54373670e-02  3.71376537e-02  3.00262496e-02
 -6.10356266e-03  3.78257483e-02 -1.16307884e-02  2.20433604e-02
 -6.11003116e-02  1.52327577e-02  1.92362461e-02  2.06531752e-02
 -3.25989574e-02 -3.34182866e-02  3.26750614e-02  2.26471610e-02
 -1.65922716e-02 -1.32516900e-04 -6.44937456e-02  6.58350391e-03
  2.32806639e-03  1.86747424e-02  9.13406909e-03  5.91993220e-02
 -1.14477174e-02 -1.92633364e-02  2.38691289e-02 -2.85223080e-03
 -5.62625937e-02  5.62440930e-03 -8.24975502e-03 -3.13492678e-02
  1.10710792e-01 -9.07478929e-02 -6.35318132e-03  2.94010080e-02
  7.22245453e-03 -6.12536212e-03 -2.78526265e-02  1.76248686e-05
  6.95810420e-03 -6.77863285e-02 -2.20561447e-03 -4.80932333e-02
  3.73495668e-02  5.17821088e-02  5.34739047e-02 -5.10814018e-04
  1.18628824e-02  3.24069075e-02 -1.02261696e-02  1.69874243e-02
  5.79551272e-02 -4.35262322e-02  3.56197879e-02 -5.35238758e-02
 -7.73706287e-03 -3.98568250e-02  3.00882617e-03 -3.59213934e-03
  1.05013293e-34  5.51356338e-02 -3.68540995e-02  5.23750065e-03
  6.07440658e-02  4.69833016e-02 -2.58567687e-02  3.60422325e-03
  9.84899607e-03  3.97792123e-02 -2.66123507e-02 -4.13678028e-03] (768,)

for chapter in book.structure.chapters:
    print(len(chapter))
    # print(chapter)
    break

57

df = pd.DataFrame(
    nlp("\n".join(p.text() for p in chapter)).vector
    for chapter in book.structure.chapters
)
df["title"] = [book.title() for chapter in book.structure.chapters]
df["chapter"] = [chapter[0].meta()["number"] for chapter in book.structure.chapters]
df["chapter_title"] = [chapter[1].text() for chapter in book.structure.chapters]
df.set_index("chapter")

	0	1	2	3	4	5	6	7	8	9	...	760	761	762	763	764	765	766	767	title	chapter_title
chapter
1	0.064237	0.003603	-0.024945	-0.028241	0.002701	0.018099	-0.005965	0.015462	-0.042051	-0.037677	...	-0.012405	-0.005935	-0.019373	-0.016045	-0.027680	0.019734	-0.022243	-0.005562	Moon Pool, The	The Thing on the Moon Path
2	0.038836	-0.000777	-0.006440	-0.007490	-0.008980	0.035333	-0.001551	0.048928	0.001041	-0.056604	...	0.005408	-0.001694	-0.032599	0.002235	-0.008573	0.043800	-0.044537	-0.015627	Moon Pool, The	“Dead! All Dead!”
3	0.089644	0.015077	-0.013878	0.027386	-0.009372	0.013859	-0.022711	-0.004822	0.009113	-0.015805	...	-0.046167	0.013813	-0.014336	-0.007238	-0.011715	0.054635	-0.019995	0.009324	Moon Pool, The	The Moon Rock
4	0.028582	-0.008979	-0.014775	0.071744	-0.030592	-0.003092	-0.035263	0.042193	-0.007287	-0.004629	...	0.023294	0.000696	-0.014906	0.048173	0.025052	0.078768	-0.032560	-0.012409	Moon Pool, The	The First Vanishings
5	0.029856	-0.057916	-0.005297	-0.010108	-0.011544	0.002319	-0.030722	0.019303	-0.031221	-0.007868	...	-0.020826	-0.007113	-0.014442	-0.029991	-0.014490	0.012318	-0.013880	0.004374	Moon Pool, The	Into the Moon Pool
6	0.021721	-0.016348	-0.007817	0.031324	-0.060723	0.046698	-0.034339	-0.006011	-0.027544	-0.035583	...	0.040005	0.024157	-0.025951	-0.042661	-0.015904	0.043297	-0.048282	-0.013511	Moon Pool, The	“The Shining Devil Took Them!”
7	0.010568	0.025463	0.024178	-0.047079	-0.026751	0.023079	0.027558	0.030201	-0.073289	-0.044047	...	-0.009532	-0.008402	-0.026374	0.030639	-0.035521	0.056884	-0.050481	-0.023022	Moon Pool, The	Larry O’Keefe
8	0.040986	0.040427	-0.004290	0.033151	0.009744	-0.029627	-0.043278	-0.001269	-0.047174	-0.041163	...	-0.012597	0.026173	-0.046367	0.105429	0.012775	0.015829	0.015085	-0.039988	Moon Pool, The	Olaf’s Story
9	0.052530	0.027686	-0.001898	-0.003500	-0.032793	0.000238	-0.017934	0.000603	-0.055443	-0.054767	...	-0.023459	-0.000665	-0.028174	0.063155	0.002944	0.024110	-0.033234	-0.031480	Moon Pool, The	A Lost Page of Earth
10	0.076578	-0.030307	-0.020680	-0.028371	-0.006413	0.003364	0.002874	0.006149	0.001353	0.007304	...	-0.023487	0.044204	-0.023937	0.007883	0.000587	0.048244	-0.030960	-0.025749	Moon Pool, The	The Moon Pool
11	0.019684	0.006673	0.003932	0.038921	-0.029661	0.001718	0.014010	0.028995	-0.067928	-0.034564	...	-0.020134	0.049745	-0.029202	0.019210	-0.030000	0.059965	-0.021109	-0.021934	Moon Pool, The	The Flame-Tipped Shadows
12	0.046804	0.019785	-0.005390	-0.032250	-0.023080	0.046572	-0.005174	0.065664	-0.051841	-0.058467	...	-0.031996	-0.003993	-0.023884	0.035619	-0.054821	0.055675	-0.004208	-0.049965	Moon Pool, The	The End of the Journey
13	0.074264	-0.027530	-0.003629	0.060321	-0.028649	0.037552	-0.021609	0.018178	-0.001756	-0.003828	...	0.006746	-0.000061	0.012565	0.039778	0.017345	0.009308	-0.011608	-0.007676	Moon Pool, The	Yolara, Priestess of the Shining One
14	0.044030	0.049102	0.004392	0.071895	-0.009171	0.021366	-0.001110	0.036874	0.013760	-0.071354	...	-0.011855	0.026503	-0.018445	0.025912	-0.011764	0.010966	-0.038447	-0.004601	Moon Pool, The	The Justice of Lora
15	0.068060	-0.020784	-0.018051	0.070283	-0.064832	0.043472	-0.038270	0.005739	-0.034214	-0.073387	...	-0.018694	0.017992	-0.037553	0.034515	0.001150	0.003613	-0.003100	-0.036371	Moon Pool, The	The Angry, Whispering Globe
16	0.045756	0.018277	0.009461	0.027479	0.002597	0.019850	0.004579	0.028065	-0.035750	-0.053621	...	-0.007003	-0.010956	-0.012369	0.025228	0.018563	-0.015025	0.002941	0.000863	Moon Pool, The	vs.
17	0.008292	0.024228	-0.003471	0.022439	0.000886	0.016660	-0.015838	0.017748	-0.050261	-0.076677	...	-0.013249	-0.004866	-0.012309	0.022411	0.004002	0.029586	0.005177	-0.004988	Moon Pool, The	The Leprechaun
18	-0.011085	-0.012101	-0.008845	0.080363	-0.006041	-0.004856	0.018779	0.034874	-0.032863	-0.034960	...	-0.040037	0.006137	-0.026947	-0.077186	-0.002093	0.013223	-0.051163	-0.029977	Moon Pool, The	The Amphitheatre of Jet
19	0.041277	-0.001336	-0.002407	0.028163	0.016887	-0.006581	-0.022635	0.015977	-0.041719	-0.011096	...	-0.023721	-0.025121	-0.006433	0.060968	0.011272	0.007868	-0.028026	-0.034190	Moon Pool, The	The Madness of Olaf
20	0.055655	0.011646	0.011576	0.053100	-0.003897	0.035930	0.016899	0.050840	0.023765	-0.049657	...	-0.015036	0.024212	-0.029889	0.028493	-0.014696	0.027860	-0.031587	-0.044249	Moon Pool, The	The Tempting of Larry
21	0.046194	0.005564	0.005479	0.042590	0.007213	0.037490	0.019260	0.023295	0.017506	-0.045237	...	-0.019125	0.023002	-0.042083	0.001311	0.011371	0.057744	-0.027158	-0.020763	Moon Pool, The	Larry’s Defiance
22	0.011239	0.001086	-0.001630	0.048478	-0.061186	-0.002877	0.032123	-0.007537	-0.100034	-0.046582	...	-0.038520	0.016293	-0.019491	-0.085275	-0.010947	0.005053	-0.012698	-0.025354	Moon Pool, The	The Casting of the Shadow
23	0.035339	0.015220	0.015888	0.031553	-0.025045	-0.051204	-0.017331	-0.010075	-0.032181	-0.048954	...	-0.056727	0.030971	0.005715	0.051151	-0.018875	0.034907	-0.046242	-0.020239	Moon Pool, The	Dragon Worm and Moss Death
24	0.079176	-0.038791	-0.020345	0.012907	-0.030331	0.003220	-0.033693	0.027084	-0.063602	-0.072106	...	0.005900	0.026290	-0.024001	-0.022364	0.026912	0.035587	-0.027175	-0.024836	Moon Pool, The	The Crimson Sea
25	-0.001657	-0.013057	-0.032553	0.087987	-0.034539	-0.004683	0.007095	0.010177	-0.038538	-0.020782	...	-0.056781	0.011871	-0.009365	-0.035336	-0.026849	-0.002551	-0.036237	-0.018739	Moon Pool, The	The Three Silent Ones
26	0.088130	0.004673	-0.003805	0.027967	0.004875	0.036757	0.006345	0.016427	-0.031302	-0.061639	...	0.001287	-0.016642	-0.025152	0.040480	-0.017742	0.046968	-0.020544	-0.018451	Moon Pool, The	The Wooing of Lakla
27	0.047731	0.004143	-0.015695	0.041636	-0.017311	0.012706	0.027057	0.033789	0.006002	-0.052111	...	0.015817	0.002949	-0.019567	0.013589	0.012520	0.023234	-0.002871	-0.019095	Moon Pool, The	The Coming of Yolara
28	0.041135	-0.045040	-0.000932	-0.015678	0.006839	0.000128	-0.027235	0.052401	-0.011537	-0.024691	...	-0.030791	-0.032478	-0.005531	-0.043347	-0.013823	0.049589	-0.034641	-0.006144	Moon Pool, The	In the Lair of the Dweller
29	0.045776	-0.057194	-0.031530	0.036874	-0.001509	0.037645	0.037090	0.028990	-0.030048	-0.062789	...	0.031701	0.022792	-0.041066	0.001265	0.003038	0.034733	-0.031970	-0.022536	Moon Pool, The	The Shaping of the Shining One
30	0.066470	-0.047913	-0.025220	0.001890	-0.010575	0.021573	0.039357	-0.016608	-0.022150	-0.045438	...	0.009846	0.028412	-0.048450	-0.078110	-0.017077	0.008402	-0.013012	-0.022352	Moon Pool, The	The Building of the Moon Pool
31	0.054702	-0.014054	-0.020134	-0.011474	0.020808	0.027376	0.028521	0.033079	-0.019378	-0.071207	...	-0.018724	0.010321	-0.033606	0.004594	0.009093	0.063579	-0.002794	-0.046866	Moon Pool, The	Larry and the Frog-Men
32	0.022476	-0.010889	-0.021300	0.036753	-0.028258	0.040821	0.016443	0.033315	0.005125	-0.051011	...	0.049223	0.019451	-0.038534	-0.007764	-0.001089	0.033181	-0.009893	-0.014958	Moon Pool, The	“Your Love; Your Lives; Your Souls!”
33	0.032298	0.010583	-0.001718	0.039748	-0.001717	-0.002087	-0.023207	0.022712	-0.015331	-0.026243	...	0.021010	0.029768	-0.023817	0.043705	-0.019920	0.048464	-0.039454	-0.039283	Moon Pool, The	The Meeting of Titans
34	0.061736	-0.040893	-0.008353	0.047025	-0.009105	0.005435	-0.014353	-0.028966	-0.052570	-0.039675	...	-0.003327	0.011402	-0.014880	-0.032444	0.036213	0.003131	-0.014364	-0.032766	Moon Pool, The	The Coming of the Shining One
35	0.050529	-0.028509	-0.017161	0.056846	0.001802	0.020352	-0.012080	-0.018919	-0.016508	-0.041150	...	0.010906	0.038564	-0.023723	0.019232	0.026779	0.027357	-0.028321	-0.018579	Moon Pool, The	“Larry—Farewell!”

35 rows × 770 columns

from sklearn.metrics.pairwise import cosine_similarity

similarity_matrix = pd.DataFrame(
    cosine_similarity(df.drop(["title", "chapter", "chapter_title"], axis=1))
)
similarity_matrix.index = df["chapter_title"]
similarity_matrix.columns = df["chapter_title"]
similarity_matrix

chapter_title	The Thing on the Moon Path	“Dead! All Dead!”	The Moon Rock	The First Vanishings	Into the Moon Pool	“The Shining Devil Took Them!”	Larry O’Keefe	Olaf’s Story	A Lost Page of Earth	The Moon Pool	...	The Wooing of Lakla	The Coming of Yolara	In the Lair of the Dweller	The Shaping of the Shining One	The Building of the Moon Pool	Larry and the Frog-Men	“Your Love; Your Lives; Your Souls!”	The Meeting of Titans	The Coming of the Shining One	“Larry—Farewell!”
chapter_title
The Thing on the Moon Path	1.000001	0.605636	0.626262	0.428531	0.554514	0.483322	0.617734	0.308145	0.591134	0.665413	...	0.523852	0.378284	0.427480	0.519446	0.587610	0.512801	0.382094	0.233544	0.328349	0.358054
“Dead! All Dead!”	0.605636	1.000000	0.575851	0.645805	0.734767	0.719356	0.519233	0.474954	0.665276	0.637052	...	0.606655	0.465196	0.513423	0.562734	0.509709	0.542309	0.593097	0.358403	0.506394	0.613200
The Moon Rock	0.626262	0.575851	1.000000	0.598770	0.582890	0.557803	0.320582	0.490620	0.582127	0.631348	...	0.566880	0.454932	0.389868	0.554637	0.656345	0.497526	0.466810	0.376129	0.580580	0.512111
The First Vanishings	0.428531	0.645805	0.598770	1.000000	0.511882	0.649904	0.404054	0.466313	0.602659	0.539929	...	0.559359	0.474503	0.556738	0.576802	0.462265	0.454059	0.578725	0.404047	0.594723	0.617377
Into the Moon Pool	0.554514	0.734767	0.582890	0.511882	1.000000	0.614757	0.383966	0.383602	0.551906	0.646520	...	0.478719	0.426616	0.519273	0.472779	0.565323	0.433763	0.486933	0.303842	0.447624	0.473360
“The Shining Devil Took Them!”	0.483322	0.719356	0.557803	0.649904	0.614757	1.000000	0.462496	0.491711	0.641435	0.632647	...	0.552671	0.473274	0.484346	0.561280	0.481544	0.479014	0.608385	0.393346	0.616364	0.636859
Larry O’Keefe	0.617734	0.519233	0.320582	0.404054	0.383966	0.462496	1.000000	0.291213	0.621495	0.575065	...	0.549711	0.355948	0.364341	0.406636	0.329174	0.471893	0.423093	0.333213	0.318331	0.351370
Olaf’s Story	0.308145	0.474954	0.490620	0.466313	0.383602	0.491711	0.291213	1.000000	0.595645	0.500984	...	0.472752	0.386482	0.287705	0.442590	0.343329	0.433966	0.509693	0.431775	0.521639	0.540777
A Lost Page of Earth	0.591134	0.665276	0.582127	0.602659	0.551906	0.641435	0.621495	0.595645	1.000000	0.654374	...	0.666253	0.560707	0.521026	0.626104	0.550797	0.648053	0.609368	0.430290	0.620957	0.628388
The Moon Pool	0.665413	0.637052	0.631348	0.539929	0.646520	0.632647	0.575065	0.500984	0.654374	1.000000	...	0.582000	0.436757	0.444891	0.534134	0.573506	0.580877	0.506372	0.457995	0.485995	0.530405
The Flame-Tipped Shadows	0.469498	0.588067	0.497807	0.568677	0.508629	0.603012	0.488941	0.544530	0.641140	0.627330	...	0.583777	0.490091	0.457172	0.598570	0.520126	0.528806	0.591339	0.500273	0.614589	0.560969
The End of the Journey	0.570227	0.607563	0.364308	0.493820	0.520204	0.582058	0.636831	0.314581	0.617962	0.608292	...	0.660202	0.458970	0.484691	0.532451	0.401394	0.675355	0.519542	0.355327	0.381092	0.493097
Yolara, Priestess of the Shining One	0.384465	0.383194	0.527154	0.529316	0.495325	0.489378	0.285650	0.343893	0.450851	0.484719	...	0.486815	0.655793	0.369646	0.633046	0.593660	0.333905	0.553298	0.376607	0.629971	0.472373
The Justice of Lora	0.253609	0.418952	0.383046	0.412965	0.336908	0.470585	0.296364	0.486389	0.489265	0.408073	...	0.549355	0.586827	0.281411	0.557450	0.422549	0.533500	0.638648	0.508789	0.564943	0.597277
The Angry, Whispering Globe	0.398156	0.518900	0.504844	0.531404	0.434987	0.547493	0.300490	0.374039	0.573421	0.420804	...	0.595208	0.665020	0.347154	0.681773	0.602107	0.530071	0.663619	0.419301	0.606809	0.627735
vs.	0.339168	0.437571	0.395311	0.442435	0.391415	0.467581	0.391373	0.440031	0.495171	0.500646	...	0.543347	0.680484	0.333293	0.517649	0.462769	0.510525	0.581612	0.536052	0.569597	0.523531
The Leprechaun	0.503686	0.606965	0.523791	0.502376	0.519976	0.580068	0.581428	0.511628	0.670781	0.559222	...	0.696877	0.747792	0.425336	0.692839	0.570896	0.616004	0.746320	0.483627	0.597063	0.625904
The Amphitheatre of Jet	0.339644	0.404705	0.406448	0.440496	0.390366	0.423188	0.363048	0.327391	0.466297	0.354984	...	0.530281	0.736463	0.332021	0.631967	0.567633	0.472112	0.634167	0.476750	0.597233	0.564457
The Madness of Olaf	0.363791	0.482890	0.436706	0.482329	0.403479	0.447638	0.291748	0.620144	0.592914	0.484278	...	0.569696	0.633734	0.419465	0.592047	0.531139	0.524628	0.569069	0.561366	0.592049	0.565604
The Tempting of Larry	0.429462	0.539910	0.407466	0.462698	0.446141	0.509360	0.471686	0.412655	0.598803	0.466433	...	0.640504	0.787262	0.305431	0.714091	0.602643	0.637346	0.747147	0.458763	0.546010	0.643805
Larry’s Defiance	0.394112	0.550261	0.445104	0.507789	0.416036	0.555938	0.478520	0.492958	0.577812	0.502455	...	0.666682	0.728466	0.320023	0.716624	0.567972	0.669238	0.809751	0.539944	0.581362	0.712860
The Casting of the Shadow	0.375156	0.454294	0.515380	0.460598	0.521247	0.525705	0.322549	0.333484	0.484741	0.424852	...	0.461251	0.586629	0.437415	0.593387	0.601019	0.397662	0.567353	0.405533	0.660347	0.512434
Dragon Worm and Moss Death	0.291890	0.454213	0.422762	0.463920	0.367615	0.464382	0.351587	0.422652	0.502194	0.505095	...	0.502732	0.436779	0.373524	0.417248	0.361597	0.483289	0.443911	0.423769	0.500718	0.530123
The Crimson Sea	0.519501	0.591393	0.515109	0.545350	0.516875	0.638390	0.478364	0.480868	0.715450	0.601700	...	0.731570	0.677278	0.403169	0.760545	0.639036	0.737810	0.746536	0.439571	0.677716	0.758767
The Three Silent Ones	0.391471	0.420331	0.451075	0.511471	0.469253	0.433401	0.369912	0.266794	0.480353	0.373649	...	0.535604	0.542797	0.438333	0.584681	0.535318	0.470804	0.554196	0.349819	0.523553	0.519619
The Wooing of Lakla	0.523852	0.606655	0.566880	0.559359	0.478719	0.552671	0.549711	0.472752	0.666253	0.582000	...	1.000000	0.653392	0.465755	0.711989	0.596741	0.736647	0.701980	0.532360	0.588603	0.624825
The Coming of Yolara	0.378284	0.465196	0.454932	0.474503	0.426616	0.473274	0.355948	0.386482	0.560707	0.436757	...	0.653392	1.000000	0.358593	0.752987	0.656378	0.600710	0.761910	0.515057	0.627280	0.625298
In the Lair of the Dweller	0.427480	0.513423	0.389868	0.556738	0.519273	0.484346	0.364341	0.287705	0.521026	0.444891	...	0.465755	0.358593	1.000000	0.446746	0.367540	0.351065	0.365594	0.276567	0.385300	0.329901
The Shaping of the Shining One	0.519446	0.562734	0.554637	0.576802	0.472779	0.561280	0.406636	0.442590	0.626104	0.534134	...	0.711989	0.752987	0.446746	1.000000	0.809385	0.702407	0.803509	0.444869	0.681350	0.709739
The Building of the Moon Pool	0.587610	0.509709	0.656345	0.462265	0.565323	0.481544	0.329174	0.343329	0.550797	0.573506	...	0.596741	0.656378	0.367540	0.809385	1.000001	0.593013	0.617164	0.391838	0.644737	0.574072
Larry and the Frog-Men	0.512801	0.542309	0.497526	0.454059	0.433763	0.479014	0.471893	0.433966	0.648053	0.580877	...	0.736647	0.600710	0.351065	0.702407	0.593013	1.000000	0.633639	0.505067	0.553995	0.674372
“Your Love; Your Lives; Your Souls!”	0.382094	0.593097	0.466810	0.578725	0.486933	0.608385	0.423093	0.509693	0.609368	0.506372	...	0.701980	0.761910	0.365594	0.803509	0.617164	0.633639	1.000000	0.520350	0.618160	0.791449
The Meeting of Titans	0.233544	0.358403	0.376129	0.404047	0.303842	0.393346	0.333213	0.431775	0.430290	0.457995	...	0.532360	0.515057	0.276567	0.444869	0.391838	0.505067	0.520350	1.000000	0.561306	0.482749
The Coming of the Shining One	0.328349	0.506394	0.580580	0.594723	0.447624	0.616364	0.318331	0.521639	0.620957	0.485995	...	0.588603	0.627280	0.385300	0.681350	0.644737	0.553995	0.618160	0.561306	1.000000	0.699051
“Larry—Farewell!”	0.358054	0.613200	0.512111	0.617377	0.473360	0.636859	0.351370	0.540777	0.628388	0.530405	...	0.624825	0.625298	0.329901	0.709739	0.574072	0.674372	0.791449	0.482749	0.699051	1.000000

35 rows × 35 columns

import plotly.express as px

fig = px.imshow(similarity_matrix, text_auto=True, width=800, height=800)
fig.show()

paragraph_matrix = pd.DataFrame(nlp(p.text()).vector for p in book.paragraphs())
paragraph_matrix

	0	1	2	3	4	5	6	7	8	9	...	758	759	760	761	762	763	764	765	766	767
0	0.011812	0.048559	-0.028817	-0.031490	-0.001718	0.000307	-0.050142	-0.001226	0.023657	0.039092	...	-0.036854	0.005238	0.060744	0.046983	-0.025857	0.003604	0.009849	0.039779	-0.026612	-0.004137
1	0.098685	-0.064736	-0.004773	0.004408	-0.049093	-0.010950	-0.028957	-0.006544	-0.037166	-0.007321	...	-0.017423	-0.023199	-0.051821	0.007272	0.028012	-0.011869	-0.052609	-0.021378	0.021478	-0.035154
2	-0.001934	0.008811	-0.025983	-0.010465	-0.003624	0.024301	0.000738	0.017132	-0.031279	-0.020992	...	0.001276	-0.035136	0.042075	-0.062756	-0.035009	0.050811	-0.036904	0.031963	-0.011701	-0.014053
3	0.017347	0.013119	-0.019933	-0.029803	0.025785	0.014013	-0.044746	0.015682	-0.066624	-0.027479	...	-0.025100	-0.024532	-0.032040	-0.005989	-0.025941	0.023550	-0.010752	0.011675	-0.024839	-0.002938
4	0.010420	0.035712	-0.033145	0.003485	0.012280	0.017478	-0.009876	-0.011734	-0.032781	-0.011936	...	-0.000131	-0.012147	-0.028610	0.034134	-0.027233	0.002801	-0.004194	0.035050	-0.035219	0.000164
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
2488	-0.017312	-0.000090	-0.021702	-0.032130	-0.023085	-0.013129	-0.035856	-0.009385	-0.003892	-0.014101	...	-0.046348	-0.030476	0.031057	0.010700	-0.004123	-0.021091	-0.000242	0.040705	-0.037479	-0.003485
2489	0.025001	-0.060220	-0.013184	0.024454	-0.017393	-0.032590	0.083316	-0.002492	0.058806	0.024633	...	-0.085195	-0.013452	-0.010259	-0.001477	0.008862	0.051493	-0.006845	0.026401	-0.032194	0.046620
2490	0.060939	-0.035463	0.000965	-0.045513	-0.024586	-0.009807	0.032109	-0.006638	0.003903	-0.004694	...	-0.054134	0.010288	0.051497	0.018566	0.009406	-0.043220	0.018144	-0.013014	0.001503	0.009967
2491	0.059070	0.035528	-0.016937	0.046333	0.034918	-0.014577	-0.011248	0.003643	0.029094	0.013765	...	-0.033656	0.018236	-0.064931	-0.015065	0.007741	-0.026861	-0.041528	-0.018617	-0.050233	0.023588
2492	-0.030971	0.034895	0.015832	-0.070395	0.060452	-0.015055	-0.022913	-0.013605	-0.005612	0.009726	...	-0.054365	0.078977	-0.000273	0.015187	0.020912	-0.053088	0.000090	0.032087	-0.022363	-0.023179

2493 rows × 768 columns

communities = community_detection(
    paragraph_matrix.to_numpy(), min_community_size=5, threshold=0.75
)
paragraphs = list(book.paragraphs())
# Print for all clusters the top 3 and bottom 3 elements
for i, cluster in enumerate(communities):
    print("\nCluster {}, #{} Elements ".format(i + 1, len(cluster)))
    for paragraph_id in cluster[0:3]:
        print("\t", paragraphs[paragraph_id].text())
    print("\t", "...")
    for paragraph_id in cluster[-3:]:
        print("\t", paragraphs[paragraph_id].text())

Cluster 1, #13 Elements 
	 XXXII
	 XXXIII
	 XXXI
	 ...
	 XXVI
	 XXIII
	 XXIV

Cluster 2, #11 Elements 
	 Below us the Shining One had paused—spiralling, swirling, vibrant with all its transcendent, devilish beauty; had paused and was contemplating us. Now I could see clearly that nucleus, that core shot through with flashing veins of radiance, that ever-shifting shape of glory through the shroudings of shimmering, misty plumes, throbbing lacy opalescences, vaporous spirallings of prismatic phantom fires. Steady over it hung the seven little moons of amethyst, of saffron, of emerald and azure and silver, of rose of life and moon white. They poised themselves like a diadem—calm, serene, immobile—and down from them into the Dweller, piercing plumes and swirls and spirals, ran countless tiny strands, radiations, finer than the finest spun thread of spider’s web, gleaming filaments through which seemed to run—power—from the seven globes; like—yes, that was it—miniatures of the seven torrents of moon flame that poured through the septichromatic, high crystals in the Moon Pool’s chamber roof.
	 “When it was done they moved up the path, clustering within the Moon Pool Chamber. Moon flame streamed through the seven globes, poured down upon the pool; they saw mists arise, embrace, and become one with the moon flame—and then up through Moon Pool, shaping itself within the mists of light, whirling, radiant—the Shining One!
	 Through all the incessant but strangely ordered movement of the—thing—these lights held firm and steady. They were seven—like seven little moons. One was of a pearly pink, one of a delicate nacreous blue, one of lambent saffron, one of the emerald you see in the shallow waters of tropic isles; a deathly white; a ghostly amethyst; and one of the silver that is seen only when the flying fish leap beneath the moon.
	 ...
	 The rose-light died; all that immense chamber was black, save for the circle of the glowing spheres. Within this their milky radiance grew brighter—brighter. The song whispered away. A throbbing arpeggio dripped from the harps, and as the notes pulsed out, up from the globes, as though striving to follow, pulsed with them tips of moon-fire cones, such as I had seen before Yolara’s altar. Weirdly, caressingly, compellingly the harp notes throbbed in repeated, re-repeated theme, holding within itself the same archaic golden quality I had noted in the singing. And over the moon flame pinnacles rose higher!
	 “It revealed to them that the force that is within moon flame is kin to the force that is within it, for the chamber of its birth was the chamber too of moon birth and into it went the subtle essence and powers that flow in that Earth child; and it taught them how to make that which fills what you call the Moon Pool whose opening is close behind its Veil hanging upon the gleaming cliffs.
	 “When this was done it taught them how to make and how to place the seven lights through which moon flame streams into Moon Pool—the seven lights that are kin to its own seven orbs even as its fires are kin to moon fires—and which would open for it a path that it could tread. And all this the Taithu did, working so secretly that neither those of their race whose faces were set against the Shining One nor the busy men above know aught of it.

Cluster 3, #10 Elements 
	 Lakla raised her head; swept back the silken tent of her hair and gazed at me with eyes misty from weeping. The frog-woman crept to her side; gazed down upon Larry; spoke—spoke—to the Golden Girl in a swift stream of the sonorous, reverberant monosyllables; and Lakla answered her in kind. The webbed digits swept over O’Keefe’s face, felt at his heart; she shook her head and moved ahead of us up the passage.
	 “Put it down, Lakla,” the distress in O’Keefe’s voice was deep. Lakla laughed mischievously, caught the real fear for her in his eyes; opened her hand, gave another faint call—and back it flew to its fellows.
	 But if the frog-woman regarded us all, not so did the maiden of the rosy wall. Her eyes were fastened upon Larry, drinking him in with extraordinary intentness. She was tall, far over the average of women, almost as tall, indeed, as O’Keefe himself; not more than twenty years old, if that, I thought. Abruptly she leaned forward, the golden eyes softened and grew tender; the red lips moved as though she were speaking.
	 ...
	 She spoke to the giant frog-man. He wheeled behind her as she turned, facing the priestess, club upraised, fangs glistening. His troop moved not a jot, spears held high. Lakla began to pass slowly—almost, I thought, tauntingly—and as she reached the portal Larry leaped from the dais.
	 “Look!” she cried. “Look! Why, even she does not believe!” Her voice grew silk of silver—merciless, cruel. “Now am I minded to send another answer to the Silent Ones. Yea! But not by you, Lakla; by these”—she pointed to the frog-men, and, swift as light, her hand darted into her bosom, bringing forth the little shining cone of death.
	 At the door was a booming. Into the chamber rushed a dozen of the frog-men. While some guarded the entrances, others leaped straight to us, and forming a circle about us began to strike with talons and spurs at unseen things that screamed and sought to escape. Now here and there about the blue rugs great stains of blood appeared; heads of dwarfs, torn arms and gashed bodies, half occulted, half revealed. And at last the priestess lay silent, vanquished, white body gleaming with that uncanny—fragmentariness—from her torn robes. Then O’Keefe reached down, drew Lakla from her. Shakily, Yolara rose to her feet. The handmaiden, face still blazing with wrath, stepped before her; with difficulty she steadied her voice.

Cluster 4, #9 Elements 
	 “Not unforeseen by the three was this, Yolara,” she replied. “And did you speak as you have spoken then was I bidden to say this to you.” Her voice deepened. “Three tal have you to take counsel, Yolara. And at the end of that time these things must you have determined—either to do or not to do: first, send the strangers to the Silent Ones; second, give up, you and Lugur and all of you, that dream you have of conquest of the world without; and, third, forswear the Shining One! And if you do not one and all these things, then are you done, your cup of life broken, your wine of life spilled. Yea, Yolara, for you and the Shining One, Lugur and the Nine and all those here and their kind shall pass! This say the Silent Ones, ‘Surely shall all of ye pass and be as though never had ye been!’ ”
	 “This, they say, is what is to happen. First will come upon us Lugur and Yolara with all their host. Because of fear the Shining One will lurk behind within its lair; for despite all, the Dweller does dread the Three, and only them. With this host the Voice and the priestess will strive to conquer. And if they do, then will they be strong enough, too, to destroy us all. For if they take the abode they banish from the Dweller all fear and sound the end of the Three.
	 “This they—say—the Silent Ones,” she gasped and then all the courage of her came back. “O heart of mine!” she whispered to Larry, gazing deep into his eyes, his anxious face cupped between her white palms. “This they say—that should the Shining One come to succour Yolara and Lugur, should it conquer its fear—and—do this—then is there but one way left to destroy it—and to save your world.”
	 ...
	 “Yolara,” she said, “you have defied the Silent Ones, you have desecrated their abode, you came to slay these men who are the guests of the Silent Ones and me, who am their handmaiden—why did you do these things?”
	 “Yolara”—her voice was low—“of no use is it to question me. I am but the messenger of the Silent Ones. And one thing only am I bidden to ask you—do you deliver to me the three strangers?”
	 “Those are great words—great words indeed, choya,” shrilled Yolara at last; and again Lakla winced beneath the word. “Lo, for laya upon laya, the Shining One has been freed from the Three; and for laya upon laya they have sat helpless, rotting. Now I ask you again—whence comes their power to lay their will upon me, and whence comes their strength to wrestle with the Shining One and the beloved of the Shining One?”

Cluster 5, #8 Elements 
	 “I listened while the Three spoke to you,” she said. “Now the shaping of the Shining One had been a long, long travail and time had flown over the outer world laya upon laya. For a space the Shining One was content to dwell here; to be fed with the foods of light: to open the eyes of the Three to mystery upon mystery and to read for them facet after facet of the gem of truth. Yet as the tides of consciousness flowed through it they left behind shadowings and echoes of their burdens; and the Shining One grew stronger, always stronger of itself within itself. Its will strengthened and now not always was it the will of the Three; and the pride that was woven in the making of it waxed, while the love for them that its creators had set within it waned.
	 “We crowned the Shining One with the seven orbs of light which are the channels between it and the sentience we sought to make articulate, the portals through which flow its currents and so flowing, become choate, vocal, self-realizant within our child.
	 At its touch there was an infinitely dreadful shrinking and, it seemed, a simultaneous hurling of herself into its radiance. As it wrapped its swirls around her, permeated her—the crystal chorus burst forth—tumultuously; through and through her the radiance pulsed. Began then that infinitely dreadful, but infinitely glorious, rhythm they called the dance of the Shining One. And as the girl swirled within its sparkling mists another and another flew into its embrace, until, at last, the dais was an incredible vision; a mad star’s Witches’ Sabbath; an altar of white faces and bodies gleaming through living flame; transfused with rapture insupportable and horror that was hellish—and ever, radiant plumes and spirals expanding, the core of the Shining One waxed—growing greater—as it consumed, as it drew into and through itself the life-force of these lost ones!
	 ...
	 “Now have you beheld,” said Lakla, “and well you trod the road. And now shall you hear, even as the Silent Ones have commanded, what the Shining One is—and how it came to be.”
	 The Shaping of the Shining One
	 “Watching we learned, and learning we formed that ye term the Dweller, which those without name—the Shining One. Within the Universal Mother we shaped it, to be a voice to tell us her secrets, a lamp to go before us lighting the mysteries. Out of the ether we fashioned it, giving it the soul of light that still ye know not nor perhaps ever may know, and with the essence of life that ye saw blossoming deep in the abyss and that is the pulse of Earth heart we filled it. And we wrought with pain and with love, with yearning and with scorching pride and from our travail came the Shining One—our child!

Cluster 6, #7 Elements 
	 “I have said my gaze was fixed. It was. But from the side, peripherally, it took in a part of the far wall of the outer enclosure. Ages seemed to pass and a radiance stole along it. Soon drifted into sight the figure that was Stanton. Far away he was—on the gigantic wall. But still I could see the shining spirals whirling jubilantly around and through him; felt rather than saw his tranced face beneath the seven moons. A swirl of crystal notes, and he had passed. And all the time, as though from some opened well of light, the courtyard gleamed and sent out silver fires that dimmed the moonrays, yet seemed strangely to be a part of them.
	 “The light that filled and surrounded him had a nucleus, a core—something shiftingly human shaped—that dissolved and changed, gathered itself, whirled through and beyond him and back again. And as its shining nucleus passed through him Stanton’s whole body pulsed radiance. As the luminescence moved, there moved above it, still and serene always, seven tiny globes of seven colors, like seven little moons.
	 “Stanton was in the range of my fixed vision. I watched him leap up the steps and move toward the gateway. The curdled radiance seemed to await him. He stepped into it—and was lost to my sight.
	 ...
	 “A faint glow in the sky heralded the moon. Stanton and I took our places. The moon dawn increased rapidly; the disk swam up, and in a moment it was shining in full radiance upon ruins and sea.
	 “At last the moon neared the horizon. There came a louder burst of sound; the second, and last, cry of Stanton, like an echo of his first! Again the soft sighing from the inner terrace. Then—utter silence!
	 “The minutes crept by. The darkness lessened and through the breaches of the terrace I watched the far sky softly lighten. With the first pale flush the silence of the place intensified. It deepened; became unbearably—expectant. The moon rose, showed the quarter, the half, then swam up into full sight like a great bubble.

Cluster 7, #7 Elements 
	 Calmness came back to me after I had made this decision. And when I went up on deck I knew that I had been right. They had not seen the Dweller. They were still discussing the darkening of the ship, talking of dynamos burned out, wires short circuited, a half dozen explanations of the extinguishment. Not until noon was Throckmartin’s absence discovered. I told the captain that I had left him early in the evening; that, indeed, I knew him but slightly, after all. It occurred to none to doubt me, or to question me minutely. Why should it have? His strangeness had been noted, commented upon; all who had met him had thought him half mad. I did little to discourage the impression. And so it came naturally that on the log it was entered that he had fallen or leaped from the vessel some time during the night.
	 Deep in thought, subconsciously with relief, I watched the shoreline sink behind; welcomed the touch of the wind of the free seas. I had hoped, and within the hope was an inexplicable shrinking that I would meet Throckmartin at lunch. He did not come down, and I was sensible of deliverance within my disappointment. All that afternoon I lounged about uneasily but still he kept to his cabin—and within me was no strength to summon him. Nor did he appear at dinner.
	 Conviction of my appalling helplessness was complete. The ensemble of the vessel from captain to cabin boy was, to put it conservatively, average. None, I knew, save Throckmartin and myself had seen the first apparition of the Dweller. Had they witnessed the second? I did not know, nor could I risk speaking, not knowing. And not seeing, how could they believe? They would have thought me insane—or worse; even, it might be, his murderer.
	 ...
	 My first act was to spring to the open port. The coma had lasted hours, for the moon was now low in the west! I ran to the door to sound the alarm. It resisted under my frantic hands; would not open. Something fell tinkling to the floor. It was the key and I remembered then that Throckmartin had turned it before we began our vigil. With memory a hope died that I had not known was in me, the hope that he had escaped from the cabin, found refuge elsewhere on the ship.
	 To the first two questions I answer frankly—I did not dare. And this reluctance, this inhibition, every man jealous of his scientific reputation will understand. The story of Throckmartin, the happenings I had myself witnessed, were incredible, abnormal, outside the facts of all known science. I shrank from the inevitable disbelief, perhaps ridicule—nay, perhaps even the graver suspicion that had caused me to seal my lips while on the ship. Why I myself could only half believe! How then could I hope to convince others?
	 He lay upon the berth. I sat thinking. I came to myself with a guilty start. I had completely lost myself in my deep preoccupation. What time was it? I looked at my watch and jumped to the porthole. It was full moonlight; the orb had been up for fully half an hour. I strode over to Throckmartin and shook him by the shoulder.

Cluster 8, #7 Elements 
	 Yolara crept close, just beyond the reach of its spirals. She murmured—and the Dweller bent toward her, its seven globes steady in their shining mists, as though listening. It drew erect once more, resumed its doubtful scrutiny. Yolara’s face darkened; she turned abruptly, spoke to a captain of her guards. A dwarf raced back between the palisades of dead-alive.
	 And again it ran swiftly with its film of colours, darkened, and shone rosy once more. From without there came a rustle of many feet upon the rugs. Yolara pressed a slender hand upon the base of the pedestal of the globe beside her. Abruptly the light faded from all, and on the same instant the four walls of blackness vanished, revealing on two sides the lovely, unfamiliar garden through the guarding rows of pillars; at our backs soft draperies hid what lay beyond; before us, flanked by flowered screens, was the corridor through which we had entered, crowded now by the green dwarfs of the great hall.
	 Abruptly the harpings ceased; the moon fires shuddered, fell, and began to sweep back into the crystal globes; Yolara’s swaying form grew rigid, every atom of it listening. She threw aside the veiling cloud of hair, and in the gleam of the last retreating spirals her face glared out like some old Greek mask of tragedy.
	 ...
	 The Dweller paused, seemed to scan the island of the Silent Ones half doubtfully; then slowly, stately, it drifted out upon the bridge. Closer it drew; behind it glided Yolara at the head of a company of her dwarfs, and at her side was the hag of the Council whose face was the withered, shattered echo of her own.
	 Yes, drew back—and back with it stepped Yolara, the doubt in her eyes deepening. Onward paced the handmaiden and the O’Keefe—and step by step, as they advanced, the Dweller withdrew; its bell notes chiming out, puzzled questioning—half fearful!
	 For a moment it paused, poised itself, and then came whirling down the flower path to its priestess, slowly, ever more slowly. It hovered for a moment between the woman and the dwarf, as though contemplating them; turned to her with its storm of tinklings softened, its murmurings infinitely caressing. Bent toward it, Yolara seemed to gather within herself pulsing waves of power; she was terrifying; gloriously, maddeningly evil; and as gloriously, maddeningly heavenly! Aphrodite and the Virgin! Tanith of the Carthaginians and St. Bride of the Isles! A queen of hell and a princess of heaven—in one!

Cluster 9, #6 Elements 
	 “And then from the moonlight flooding us there dripped down on me a great drowsiness. Sleep seemed to seep from the rays and fall upon my eyes, closing them—closing them inexorably. Edith’s hand in mine relaxed. Stanton’s head fell upon his breast and his body swayed drunkenly. I tried to rise—to fight against the profound desire for slumber that pressed on me.
	 “ ‘Stanton, what do you see?’ I called cautiously. He waved a silencing hand. I turned my head to look at Edith. A shock ran through me. She lay upon her side. Her face, grotesque with its nose and mouth covered by the respirator, was turned full toward the moon. She was again in deepest sleep!
	 “Dawn was breaking when I wakened. Recollection rushed back; I thrust a panic-stricken hand out toward Edith; touched her and my heart gave a great leap of thankfulness. She stirred, sat up, rubbing dazed eyes. Stanton lay on his side, back toward us, head in arms.
	 ...
	 “For a dozen heart beats there was silence. Then a rain of tinklings that set the pulses racing with joy and at once checked them with tiny fingers of ice—and ringing through them Stanton’s voice from the courtyard—a great cry—a scream—filled with ecstasy insupportable and horror unimaginable! And once more there was silence. I strove to burst the bonds that held me. I could not. Even my eyelids were fixed. Within them my eyes, dry and aching, burned.
	 “The moon waned; appeared crescent in the west; waxed slowly toward the full. Before the men left us they literally prayed us to accompany them. Their importunities only made us more eager to see what it was that, we were now convinced, they wanted to conceal from us. At least that was true of Stanton and myself. It was not true of Edith. She was thoughtful, abstracted—reluctant.
	 “As I turned again to call to Stanton, my eyes swept the head of the steps and stopped, fascinated. For the moonlight had thickened. It seemed to be—curdled—there; and through it ran little gleams and veins of shimmering white fire. A languor passed through me. It was not the ineffable drowsiness of the preceding night. It was a sapping of all will to move. I tried to cry out to Stanton. I had not even the will to move my lips. Goodwin—I could not even move my eyes!

Cluster 10, #6 Elements 
	 “Larry!” I cried, dazed.
	 “Larry!” I cried. “Here! Quick!”
	 “Larry,” I cried, “are you going crazy?”
	 ...
	 “Larry!” I cried. “Is he dead?”
	 “Larry!” I said.
	 “Larry!” I groaned. “Lakla!”

Cluster 11, #5 Elements 
	 “ ‘I thought of that,’ he answered, ‘and I wouldn’t dare,’ he added soberly enough. And even as I had spoken there came to me the same feeling that he had expressed. It was as though something passed out of the grey rock that struck my heart as a hand strikes an impious lip. We turned away—uneasily, and faced Thora coming through a breach on the terrace.
	 “And as I fought, Thora raised her head as though listening; and turned toward the gateway. There was infinite despair in her face—and expectancy. I tried again to rise—and a surge of sleep rushed over me. Dimly, as I sank within it, I heard a crystalline chiming; raised my lids once more with a supreme effort.
	 “ ‘I would like to see the stone,’ she said. ‘Charles, you stay here with Thora.’ We passed through the outer court silently—and stood before the rock. She touched it, drew back her hand as I had; thrust it forward again resolutely and held it there. She seemed to be listening. Then she turned to me.
	 ...
	 “ ‘I would like to see the stone,’ she said. ‘Charles, you stay here with Thora.’ We passed through the outer court silently—and stood before the rock. She touched it, drew back her hand as I had; thrust it forward again resolutely and held it there. She seemed to be listening. Then she turned to me.
	 “ ‘Thora was taken,’ was all I could say to Stanton. Together we went to my wife, now standing beside the great stone steps, looking up fearfully at the gateway into the terraces. There I told them what I had seen before sleep had drowned me. And together then we ran up the stairs, through the court and to the grey rock.
	 “The grey rock was a door! And it had opened and Thora had passed through it!

Cluster 12, #5 Elements 
	 Then Lakla raised her hands, pressed down Larry’s head, kissed him between the eyes, drew herself with a trembling little laugh from his embrace.
	 “They come!” cried Lakla, the light of battle in her eyes. Larry drew her to him, raised her in his arms, kissed her.
	 Lakla gave a little cry; made a motion toward the two. But Larry drew her head down against his breast, hiding her eyes; then fastened his own upon the pair, white-faced, stern.
	 ...
	 Lakla gave a little cry; made a motion toward the two. But Larry drew her head down against his breast, hiding her eyes; then fastened his own upon the pair, white-faced, stern.
	 Lakla had been listening again. She turned, thrust out hands to Larry, a wild hope in her eyes—and yet a hope half shamed.
	 “Well, then,” murmured Lakla, “go you, Larry and Goodwin, with Kra and Gulk, and let them minister to you. After, sleep a little—for not soon will Rador and Olaf return. And let me feel your lips before you go, Larry—darlin’!” She covered his eyes caressingly with her soft little palms; pushed him away.

Cluster 13, #5 Elements 
	 “Lead!” the handmaiden was appalled. “You lead, Larry? Why you are to stay with Goodwin and with me—up there, there we can watch.”
	 “Marry!” cried the handmaiden incredulously. “Marry us? Why, Larry dear, we are married!”
	 “Larry darlin’,” said the handmaiden. “The Silent Ones summon us!”
	 ...
	 “Larry darlin’,” said the handmaiden. “The Silent Ones summon us!”
	 “Enough?” The handmaiden’s puzzlement was complete, profound. “Enough? Larry darlin’, what more could we ask?”
	 “That is a lie!” The handmaiden’s voice shook with rage. “It is a lie! But here and now he shall choose, Yolara. And if you he choose, you and he shall go forth from here unmolested—for Yolara, it is his happiness that I most desire, and if you are that happiness—you shall go together. And now, Larry, choose!”

Cluster 14, #5 Elements 
	 They spurred Lugur. Mad now as the Norseman, the red dwarf sprang. Olaf struck a blow that would have killed an ordinary man, but Lugur only grunted, swept in, and seized him about the waist; one mighty arm began to creep up toward Huldricksson’s throat.
	 Lugur leaped forward. On the instant Larry was over the low barrier between the pillars, rushing to the Norseman’s side. And even as they ran there was another wild shout from Olaf, and he hurled himself out, straight at the throat of the Dweller!
	 Lugur paused, his hand darted up, and in it was one of the silver Keth cones. But before he could flash it upon the Norseman, Larry had unlooped his robe, thrown its fold over Olaf, and, holding him with one hand away from the Shining One, thrust with the other his pistol into the dwarf’s stomach. His lips moved, but I could not hear what he said. But Lugur understood, for his hand dropped.
	 ...
	 Lugur paused, his hand darted up, and in it was one of the silver Keth cones. But before he could flash it upon the Norseman, Larry had unlooped his robe, thrown its fold over Olaf, and, holding him with one hand away from the Shining One, thrust with the other his pistol into the dwarf’s stomach. His lips moved, but I could not hear what he said. But Lugur understood, for his hand dropped.
	 “ ’Ware, Olaf!” cried O’Keefe; but Olaf did not answer. He waited until the red dwarf’s hand was close to his shoulder; and then, with an incredibly rapid movement—once before had I seen something like it in a wrestling match between Papuans—he had twisted Lugur around; twisted him so that Olaf’s right arm lay across the tremendous breast, the left behind the neck, and Olaf’s left leg held the Voice’s armoured thighs viselike against his right knee while over that knee lay the small of the red dwarf’s back.
	 “Can’t hit him without hitting Olaf,” he whispered. Lakla signalled the frog-men; they advanced toward the two—but Olaf saw them, broke the red dwarf’s hold, sent Lugur reeling a dozen feet away.

queries = torch.from_numpy(
    pd.DataFrame(
        nlp(query).vector
        for query in ["It was dark outside.", "She said she was going somewhere."]
    ).to_numpy()
)

import torch

for result in util.semantic_search(queries, paragraph_matrix.to_numpy(), top_k=5):
    for d in result:
        print(d["score"], paragraphs[d["corpus_id"]].text())
    print("=" * 80)

0.520439088344574 We were in Cimmerian darkness. I felt for my pocket-flash and recalled with distress that I had left it behind with my medicine kit when we fled from the gardens. But Rador seemed to need no light.
0.49528855085372925 And then—swiftly the moon path faded! The clouds swept over the sky as though a hand had drawn them together. Up from the south came a roaring squall. As the moon vanished what I had seen vanished with it—blotted out as an image on a magic lantern; the tinkling ceased abruptly—leaving a silence like that which follows an abrupt thunder clap. There was nothing about us but silence and blackness!
0.49466273188591003 A wave of weakness swept over me, buried me in blackness. When consciousness came back, the lights were again burning brightly.
0.4915391206741333 The Casting of the Shadow
0.47261884808540344 “The Shadow takes its toll,” he said.
================================================================================
0.5882225036621094 The woman spoke again.
0.5812108516693115 “I don’t understand,” she said hopelessly, “if you want to walk, why, of course, you shall, Larry.” She turned to me.
0.5630208849906921 “Your words shall be mine, O one I love,” she whispered. “So going or staying, I am beside you.”
0.5534455180168152 “Walk with us,” she said to me, “unless you are still weak.”
0.5468048453330994 “Follow!” he said. “She came to show us the way! Follow? I’d follow her through a thousand hells!”
================================================================================