From 40ea2cae201ecb1896b2930b57cfc008c93839e0 Mon Sep 17 00:00:00 2001
From: Dorian <interact@dorian.systems>
Date: Fri, 16 Dec 2022 15:54:09 +0100
Subject: [PATCH] custom_summarize with self made matrix

---
 README.md                                   |  10 ++-
 summa/__pycache__/edits.cpython-38.pyc      | Bin 1593 -> 3264 bytes
 summa/__pycache__/summarizer.cpython-38.pyc | Bin 4305 -> 4475 bytes
 summa/edits.py                              |  89 +++++++++++++++++++-
 summa/summarizer.py                         |   9 +-
 5 files changed, 98 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 50fc032..6bb6b57 100644
--- a/README.md
+++ b/README.md
@@ -30,4 +30,12 @@ modify the variable `wikipedia_page` in `make.py` to whatever page then
 * using the `.content` method of python wikipedia, we get **plain text plus header in wikitext**, but things like `<p>`, `<ul>`, `<blockquote>`, etc all dissapeared. see if we want to craft a version using the `.html` method of python wikipedia, but it becomes more complex because of sentence tokenisation, probably need an index to keep track of their original div nested location.
 * **opacities were remapped** to add contrast to their curves. still need to experiment with that to find some kind of nice compromise on both paper and screen ?
 
-## [EXP] recommanded
\ No newline at end of file
+## [EXP] recommanded
+
+## [EXP] custom similarity
+
+### technical note
+
+* had to build a `similarity_graph` function to get the matrix of a text. 
+* the computation of those numbers is made in the `_get_similarity` function in `summarizer.py`, basically counting the words and dividing them by length of the sentence. the numbers can vary from approx 3.5 to 0 and are not symmetrized or normalized in any way. so it feels that we can input what we want lol
+* we want to input our own matrices, so we create a `set_graph_custom_edge_weights` and `custom_summarize`.
\ No newline at end of file
diff --git a/summa/__pycache__/edits.cpython-38.pyc b/summa/__pycache__/edits.cpython-38.pyc
index e9ccb7b41fabfd0d2cb53a4bafc32480af2832f3..bf3586b620922b36b1a33d38081584b2447482d9 100644
GIT binary patch
literal 3264
zcmcImOOG4J5$>J`IUI7O)oNF(l^>vNB{36+lpG*Ac%c}62uK7FLDu$xurZpQZgEFD
zoT0j#vLp<Fz^foS<d7U~_^^A_Psv}Hs}F|%f=>Rb`B-W#<q#lEOtbn`-Cgz7SN+-A
zTF6jt|NXn<)+S@$Q)9K%(D(qK{uY&Bf@f^R2b?qE2sd*^?!dFMm-(aGpl)S93r3AW
zXyqE@=AdQedbTz?H#leI0P^|4c`G*{uMgHChobp}4K9e5Si}E@xFRlyjVF9?QM@KD
zicLtHVoO}Y*p_%*Toya%xg^&2Sm%u&aX!}JRZYoJ@j(8_XU*+#Jd`TV52LSSIy^8^
zL^?^wk1%sP$z+^IMm{vr{-a3C+{iqU8bi(fNt%ghsN(SfnwMufqe-r0oE*ganT+y6
z*s0rCXl64bDTZ>s4{5m^#e%jgl+=+aUfAJclt7n}(StZk=Q|~I=0zdd!}k$BeGZjj
zB|GMVpYS7&3p@SiyxnmsH<Nkg>v5JEZ0TibE}t=V9t)oTsW*}$jeDX{X`J^yj!oSA
ziz@EP#OPj}$Eoh_OO+h<<il}S$)p&K;#^44>vns3f<_O^jqZ*gRn4TB<R<#6P(l-6
zw1bLqbpdr^Woo04?9M;E0pTC)E)Ff6Cqc<RX7=cA|Am{<61wR{eELmPL)`SsOH+;;
z^Lx1MQ+DVnujK!B?t6Q)yLBA9b>)%wU<Idpz2f>AuUhtqQ9c>rYSs*D9V)AEH%qm_
z6~{Npn3`~1!)ed4$5-dx)dz*bZFjRGi8Fl@!eWKhgIyX+jHpXkKU2R&lHFtN1x0JL
z&D=~okgK}PhfsNNtDpF2FG>Jm>FB?qL5?>3$k7|d5xD->+_)txozgpUr+&%RyT&W=
zf8<Pk<5L_yVWm&DJ>t`vsg<=O#HXpx|AVr2+&ICg@D3eyuXNSjA=_iZ-(x1EIn%mn
zLSOyJ;m`wU)%wr7pTe%~ajfEzG*Y!k6K&f2vK_ZS%hLqVZx{Q7qxL)6sLs`&`I8{9
zhtER5`ZJtzc$2O85Z>}laVYa=jw0Vz!NR;V!xDzAVbwOIHa<;uBCkEZW}^zhwFugm
zx{YF}Vx3kMraH~Fi4oqFixXGj=W#ZXpD0x*gy}1YKv}o7q7^bw<gIEadKDq47LEIs
z9I8gNARpC0cPCI35EeRBkj43MLfk|+sW!nBsY#1G&LVA!u|=`EY?WCSs?INT%?ag<
zjjHvt76F?xeFYWcArIV^>+#UtfE2jvyoEY+)Ne7GkWm+f!eR-=Kcn$VFuDLEV$y;U
z{{W-61fw?xqYp6p1V#%hudiN%#{UT`>UFGCL=I=}ZL#qo`d6`$j?yevsd+Tx<(FZj
zQ&(3>+gnt<P1QB3$m8l4RFR`QwNsC&4)y+;DsqeZ4OJGROPs<7XLkNQCTZe^Gd^%s
z7kvpu$)XTK$okZ$|Ap!)J7y=K%rEau-KG3j*qoK_4!g%e1+4U}R4d)myX#Er^nd88
zP0V-7`U%&YpbdYAm9-r<4az|9`KVjg%ish7e}D6SU_49GL?i)L`m`&NiTbChJ%xnC
zj9~7-e-uzZ^B>{mcXVp<5tw^(3dTMLcb{;<4xP^#b^(!5&aw*n$WzUgREcgI{vP!`
z<13c6tlHQe${ZO=-TZ<!q1`;9fa|AuHvu~pqZiWeiq99z;qqT$v2|$`9K_l#r~*pT
zv~}giSyuTZYDGz|(>Qf?RU5@dr4OrcK{?Tl75TkEUEi=z2?+05L0yLmOj$x=hL#(2
z1beYJ{_6#9W1glg!`v2cIUx_7fN$^?5BR!wg{${5^24{^Q2Eav6GH2vG$AZw_yHQP
zh+z+u?@r<Uc?t>UQQI(n8I%pgw2L%SD;x84(l`#5Y2;4nt1p&mq(O70A<{`uhBl1^
zHjVth;yuNsR^ol`e*0um>0`O#`_mLo&{b_zo#sm;-v&JyRsE7G;_(?=-XY2IcXP%D
ze?Uj&s~B`$`7@%n++c~kHhb=!C5m@R$yv5Id+s}!G;=xJO^gk9pR%>9KEQa#voWNI
z*l+ba?rEzJw%SE+e>R4}?aC>%q9;goVVOM^&tOFmQFOU#XRPE{6?pn0OAfpAq_P<i
z4=A4|U86~LdL9<JRzy-Wima1F)&k1uxk$EGmN>1wYz*0F+Kg5y*;wAQ*tNG=G07>5
d2D_s|Ofq?sTxK&E0$}y$Acxh%x5BMp>nB4&=1Kqn

literal 1593
zcmb7^TW=dh6vt=w;>+4j+^CR5T;!r6x>9BH0ip<r(ibEPDHI_Kw2XHKdz-y+W=xuD
z^Md;Y_y+KpuQX3cJoPK|iE}2gYbx@<So`-mJJ)m0Z}!<>5HW0@{q=RK+Kl~8&TjM3
z`2rvP6N+S#=d9o}kCW}?ei6(<XZv}pXwN#%4$$t-B4>wrujtSE&TgSSm<`b0lacJb
z;<G)uFZ*(U*1qhXv3U5J<2Et3hqYKJ5aoGtp|Zu&D4FOqt1qc5&6Ow<qh6R~ewk=h
z8dauBV^?p!%5s@3K-5cgk3bdGuPRxUphUV9^IRomCEd_NoM>)FCQ2^U?LM^1Q6eO5
zSAo)rsow5j+p1jTnO;7frU9OF(=cz0{4ep*fMQs~3}1Wtz<83c`ET6#4QqJgul=jQ
z1k#iKE7k;0*cyAULK8OOnsa7axBu-X{G;<H_R8SggC80XzF)92Cc`skx-{mhWuhi%
zT5GIC6JX|A(>LR9F(so~fGCtvFe+BsjOJ=2M$gJD#kfb+{JBca=#e%MKS(Lno93ZG
zr9Q()AD}ESgdd|T-tgz_ckdV6;+kD^8=C4|mDis2#-H2HCig%LIe9am6iQ}dA}h#5
zIe8+Cn0x~j?W-rE6qz2+6{P1Avs4!X)UsM>ET=^GXG*x4s2BAZR9Y1UzE*NF9#8ZN
zoe8dM^tisX(Fqp8;E4M49-WtR-(ntn`Op;&Yqw!ye3DnG$n_~2n<=kxG_x%3*(lRl
zsf|E-tdBXd(RU(WsjmPkKnm{Q%=VKV%%DZ99Vd68X8l~1Hgwq8ZnD7|h?%%;1H#yL
za#1c;NW2YoooB}0M*=gGRi(%iZL0dB0;z3x8<!0o<92_u-Yu?g3jsqq*YMqhbn&fz
zABFLVcl^G8fZFkgypI}r@DO`a0%6laG`4J?pmQhN9<uFk+4gVQ4v_7D*mjf#Q=k+4
zU-WLm>fuZnkZf^tQ365m7@u7bvLed`Wajebj{g@3aSQI#YVVOyPT-K_h=i&dhwu^E
zACrto9*}%O^6wL1p5NJz@~HbLJ9+ltQ}kmWI{5c=8hg%|#zT8=TgJ`T4(LL{hbZ>A
z2313aUpS<7F(zOfnF@dk<qmDzbYH4c1J&v#1gaby(+ZKhKajn|DHNE4&z0JpXeSsK
amLsQgQdDx4t5f<)=slD^-~RmQgMR=czLo0$

diff --git a/summa/__pycache__/summarizer.cpython-38.pyc b/summa/__pycache__/summarizer.cpython-38.pyc
index cd4765386da06c2d5b308bf66204dbc8458e0226..821c63b7e666fa1029ebaff7e50f3d6261509953 100644
GIT binary patch
delta 603
zcmY+BOKTHR6vuNWugN?^Y<(7NN}*1RX-x{DMT_(iSH@Nw+%=4up+uWZJd<M4k)pM3
zGz&Q*F1m3gE~SW{z`btgGw8y#8_!8WaNz##fByI0bI&>VNAb^u)i6yh!Qc4dQS1Au
z&sIf&IxQ=g3jSWNgnldNblk4zd$Xm|@k`|}4KW$&ZJ48``UU2XM{`AhGF7z!EYLUY
zJ$#}qeSc(uBRVN8^AxAi?Y9H0(r<kOR%ykUv~>1UJXUyw?`YR}INjHyYy^>VkbJ5|
z+I_>zc-a>TPaOax`evj>sVIGrypwQp^wVg=6<V~%2+ayz)@NqobenVB!$ppV=08}X
zt}<SWxW;&Vfu5P8w%fVqxl0b<5_^tCAI&HG^Rj1Ea9uFRaCDp!y)39QVs*#uZtuF=
zK3){BBHoDWd2Oz+>+Uqee$aaqpcf8HM?==_p6qwK51L^sKtIG;Ir@K=agGL-1?TC=
zI+rn$c?CD=tMwdiNSW}K?%JiXn{uo!ILqMf4s+WOqf8&{iRm{iLzR&LO(}q?V1u=j
z!8M-hSYy^|b*{QbzibEA=}dldU82TWdFG%SVut@t?#PO|C6L?jw4fju7f5$-*fDwO
PleTS&YS63v8WjHm0i1@E

delta 514
zcmXAmOG_g`5P(w?os5&2ag-n)0zL@1v*05JMQ;)x2X&)~D~K0qGP4>?W@2@Mpn@Rm
zaZe@a-w^TS$)DiCOa6dA!pm00hWfgyy1E|SU(@eX-l6N-DPDUY+x3aulUFj}h|0!~
zeCI5>R2ZElT^*@7Wb;pgk?zbU&G7lg_&W&4WLPzrr$g%nO3&G$DS#r)+W%mMUhPK+
z=y&!ux5BxD0W9*?8${J<B)CkU*(#K2)hT#Vdw^w~MI)U%7v@doAQhNox(!&Mw!3Ep
zSz7h3Z%Z5!*f_%*t_hYIJ*%nW)2=#=a6){G;>)QpY;dJmHBWRWX<sH7>V6C8$oX(v
z;#GWpqU$mfx^lG-PtJ#Qx7AYExQg0r>M=$N#%!?CPI{DSrG0Rib{eashVNjL`^>`w
zfj^hu=3v|rj5B!je!~@U6luqw{&~-;wiqd}4NhlZjrG7pd4mdWFt@fU0pJEb_yPQ)
yH@~niS$mm!(rP8KMw6}Y8X3`c1yT*ARU8#e3g!j!ZYbY?WkFR?qot8eDEtSPfpEb9

diff --git a/summa/edits.py b/summa/edits.py
index b77fb96..b0fd654 100644
--- a/summa/edits.py
+++ b/summa/edits.py
@@ -1,18 +1,33 @@
+from os import remove
 from .pagerank_weighted import pagerank_weighted_scipy as _pagerank
 from .preprocessing.textcleaner import clean_text_by_sentences as _clean_text_by_sentences
 from .commons import build_graph as _build_graph
 from .commons import remove_unreachable_nodes as _remove_unreachable_nodes
 from .summarizer import _set_graph_edge_weights
 from .summarizer import _add_scores_to_sentences
+from .summarizer import _create_valid_graph
 
+def count_words(line):
+    return len(line.split(" "))
 
-def scored_sentences(text, language="english", split=False, additional_stopwords=None):
+def remove_shorts(sentences, words_number):
+    return [s for s in sentences if count_words(s.text) > words_number]
+
+
+# SCORED SENTENCES (output the list of sentence with their scores)
+# -------------------------------------------------------------------------------------
+
+def scored_sentences(text, words_number=0, language="english", split=False, additional_stopwords=None):
     if not isinstance(text, str):
         raise ValueError("Text parameter must be a Unicode object (str)!")
 
     # Gets a list of processed sentences.
     sentences = _clean_text_by_sentences(text, language, additional_stopwords)
 
+    # Remove sentence with less than words_number words.
+    if words_number:
+        sentences = remove_shorts(sentences, words_number)
+
     # Creates the graph and calculates the similarity coefficient for every pair of nodes.
     graph = _build_graph([sentence.token for sentence in sentences])
     _set_graph_edge_weights(graph)
@@ -33,15 +48,83 @@ def scored_sentences(text, language="english", split=False, additional_stopwords
     return sentences
 
 
-def similarity_graph(text, language="english", additional_stopwords=None):
+# SIMILARITY GRAPH (output the whole matrix/graph of similarity with weighted edges)
+# -------------------------------------------------------------------------------------
+
+def similarity_graph(text, words_number=0, language="english", additional_stopwords=None):
     if not isinstance(text, str):
         raise ValueError("Text parameter must be a Unicode object (str)!")
 
     # Gets a list of processed sentences.
     sentences = _clean_text_by_sentences(text, language, additional_stopwords)
 
+    # Remove sentence with less than words_number words.
+    if words_number:
+        sentences = remove_shorts(sentences, words_number)
+
     # Creates the graph and calculates the similarity coefficient for every pair of nodes.
     graph = _build_graph([sentence.token for sentence in sentences])
     _set_graph_edge_weights(graph)
 
-    return graph
\ No newline at end of file
+    return graph
+
+
+# CUSTOM SUMMARIZATION (self made similarity matrix)
+# -------------------------------------------------------------------------------------
+
+def set_graph_custom_edge_weights(sentences, graph, matrix):
+    # allow us to input a custom 'hand-made' matrix and feed it to pagerank
+
+    for sentence_1 in graph.nodes():
+        for sentence_2 in graph.nodes():
+
+            s1_text = sentences[sentence_1].text
+            s2_text = sentences[sentence_2].text
+
+            edge = (sentence_1, sentence_2)
+            if sentence_1 != sentence_2 and not graph.has_edge(edge):
+                similarity = matrix[s1_text][s2_text]
+                if similarity != 0:
+                    graph.add_edge(edge, similarity)
+
+    # Handles the case in which all similarities are zero.
+    # The resultant summary will consist of random sentences.
+    if all(graph.edge_weight(edge) == 0 for edge in graph.edges()):
+        _create_valid_graph(graph)
+
+
+def custom_summarize(text, matrix, ratio=0.2, words=None, language="english", split=False, scores=False, additional_stopwords=None):
+    
+    if not isinstance(text, str):
+        raise ValueError("Text parameter must be a Unicode object (str)!")
+
+    # Gets a list of processed sentences.
+    sentences = _clean_text_by_sentences(text, language, additional_stopwords)
+
+    # Creates the graph and calculates the similarity coefficient for every pair of nodes.
+    graph = _build_graph([sentence.token for sentence in sentences])
+    set_graph_custom_edge_weights(sentences, graph, matrix)
+
+    # Remove all nodes with all edges weights equal to zero.
+    _remove_unreachable_nodes(graph)
+
+    # PageRank cannot be run in an empty graph.
+    if len(graph.nodes()) == 0:
+        return [] if split else ""
+
+    # Ranks the tokens using the PageRank algorithm. Returns dict of sentence -> score
+    pagerank_scores = _pagerank(graph)
+
+    # Adds the summa scores to the sentence objects.
+    _add_scores_to_sentences(sentences, pagerank_scores)
+
+    # EDIT: return the whole sentences with scores
+    return sentences
+
+    # Extracts the most important sentences with the selected criterion.
+    # extracted_sentences = _extract_most_important_sentences(sentences, ratio, words)
+
+    # Sorts the extracted sentences by apparition order in the original text.
+    # extracted_sentences.sort(key=lambda s: s.index)
+
+    # return _format_results(extracted_sentences, split, scores)
\ No newline at end of file
diff --git a/summa/summarizer.py b/summa/summarizer.py
index 952625e..1ab296d 100644
--- a/summa/summarizer.py
+++ b/summa/summarizer.py
@@ -133,16 +133,13 @@ def summarize(text, ratio=0.2, words=None, language="english", split=False, scor
     # Adds the summa scores to the sentence objects.
     _add_scores_to_sentences(sentences, pagerank_scores)
 
-    # EDIT: return the whole sentences with scores
-    return sentences
-
     # Extracts the most important sentences with the selected criterion.
-    # extracted_sentences = _extract_most_important_sentences(sentences, ratio, words)
+    extracted_sentences = _extract_most_important_sentences(sentences, ratio, words)
 
     # Sorts the extracted sentences by apparition order in the original text.
-    # extracted_sentences.sort(key=lambda s: s.index)
+    extracted_sentences.sort(key=lambda s: s.index)
 
-    # return _format_results(extracted_sentences, split, scores)
+    return _format_results(extracted_sentences, split, scores)
 
 
 def get_graph(text, language="english"):