From 8ae06f04df2cc70bc004d388b2be917f06829595 Mon Sep 17 00:00:00 2001
From: Dorian <interact@dorian.systems>
Date: Sun, 16 Oct 2022 11:10:52 +0200
Subject: [PATCH] wikipage getter in a separate folder so its also usable from
 different experiments

---
 README.md                                    |  16 +++++-
 exp.opacity/make.py                          |  38 ++-----------
 exp.recommanded/make.py                      |  56 +++++++++++++++++++
 summa/__init__.pyc                           | Bin 0 -> 339 bytes
 summa/__pycache__/__init__.cpython-38.pyc    | Bin 342 -> 369 bytes
 summa/__pycache__/edits.cpython-38.pyc       | Bin 0 -> 1589 bytes
 summa/commons.pyc                            | Bin 0 -> 865 bytes
 summa/edits.py                               |  16 +++++-
 wikipage/__pycache__/__init__.cpython-38.pyc | Bin 0 -> 228 bytes
 wikipage/__pycache__/page.cpython-38.pyc     | Bin 0 -> 679 bytes
 wikipage/page.py                             |  25 +++++++++
 11 files changed, 115 insertions(+), 36 deletions(-)
 create mode 100644 exp.recommanded/make.py
 create mode 100644 summa/__init__.pyc
 create mode 100644 summa/__pycache__/edits.cpython-38.pyc
 create mode 100644 summa/commons.pyc
 create mode 100644 wikipage/__pycache__/__init__.cpython-38.pyc
 create mode 100644 wikipage/__pycache__/page.cpython-38.pyc
 create mode 100644 wikipage/page.py

diff --git a/README.md b/README.md
index c61a8bb..50fc032 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,12 @@
 
-## opacity experiment
+## edited summa (textrank)
+
+summa is a textrank python implementation (https://github.com/summanlp/textrank).
+it was modified under `summa/`, by adding an `summa/edits.py` files to create two new function, to access the internal process steps of textrank:
+1. `scored_sentences`: gives the list of all the sentences with their score.
+2. `similarity_graph`: gives the matrix of similarity of all the sentences in a text.
+
+## [EXP] opacity
 
 For any wikipedia page, show the text content but where every sentences has an opacity inversely proportional to its TextRank score.
 
@@ -7,17 +14,20 @@ Meaning sentences considered as _"relevant to be included in a summary of the ar
 
 ### using
 
-* textrank python implementation (https://github.com/summanlp/textrank) modified under `summa/` so it gives us all the sentences with their score.
+* edited summa (https://github.com/summanlp/textrank)
 * wikipedia python module (https://pypi.org/project/wikipedia/)
 
 ### to use
 
 modify the variable `wikipedia_page` in `make.py` to whatever page then
 
+        cd exp.opacity
         python3 make.py
 
 ### technical notes
 
 * **headers opacities** where manually recomputed has average of their section, this is justified because otherwise their break the flow of the document (their shortness seems to either put them nearly full black or white otherwise, independantly of how textrank rank the paragraphs in their associated sections)
 * using the `.content` method of python wikipedia, we get **plain text plus header in wikitext**, but things like `<p>`, `<ul>`, `<blockquote>`, etc all dissapeared. see if we want to craft a version using the `.html` method of python wikipedia, but it becomes more complex because of sentence tokenisation, probably need an index to keep track of their original div nested location.
-* **opacities were remapped** to add contrast to their curves. still need to experiment with that to find some kind of nice compromise on both paper and screen ?
\ No newline at end of file
+* **opacities were remapped** to add contrast to their curves. still need to experiment with that to find some kind of nice compromise on both paper and screen ?
+
+## [EXP] recommanded
\ No newline at end of file
diff --git a/exp.opacity/make.py b/exp.opacity/make.py
index 4d9626f..3889316 100644
--- a/exp.opacity/make.py
+++ b/exp.opacity/make.py
@@ -1,18 +1,16 @@
 from jinja2 import Template
-import os
-import wikipedia
 from markdown import markdown
-
-# importing module
 import sys
   
 # appending a path
 sys.path.append('../')
   
-# importing required module
-import summa.summarizer
-from summa.summarizer import scored_sentences
+# importing customised module
+import summa.edits
+from summa.edits import scored_sentences
 
+import wikipage
+from wikipage.page import get_wikipage
 
 # TODO:
 # * DONE: wiki header
@@ -57,30 +55,6 @@ def compress_score(s):
 
     return s
 
-
-#   wikipedia
-#   ------------------------------------------------------------------------
-
-def wikipage(pagename):
-    # get wikipedia page content by name of the page
-
-    print(pagename)
-    wikipedia.set_lang("en")
-    try:
-        results = wikipedia.search(pagename, results=1, suggestion=False)
-        try:
-            pagename = results[0]
-        except IndexError:
-            # if there is no suggestion or search results, the page doesn't exist
-            raise wikipedia.PageError(pagename)
-        return wikipedia.WikipediaPage(pagename, redirect=True, preload=True)
-    except wikipedia.exceptions.DisambiguationError as e:
-        print(e.options)
-        page = ''
-
-    return page
-
-
 #   parsing and gluing html
 #   ------------------------------------------------------------------------
 
@@ -180,7 +154,7 @@ if __name__ == '__main__':
 
     # get text from wikipedia
     print('--- WIKI ---')
-    page = wikipage(wikipedia_page)
+    page = get_wikipage(wikipedia_page)
     if not page:
         sys.exit("--- STOP ---")
     title = '<h1>'+page.title+'</h1>'
diff --git a/exp.recommanded/make.py b/exp.recommanded/make.py
new file mode 100644
index 0000000..d928084
--- /dev/null
+++ b/exp.recommanded/make.py
@@ -0,0 +1,56 @@
+from jinja2 import Template
+from markdown import markdown
+import sys
+  
+# appending a path
+sys.path.append('../')
+  
+# importing customised module
+import summa.edits
+from summa.edits import scored_sentences, similarity_graph
+
+import wikipage
+from wikipage.page import get_wikipage
+
+wikipedia_page = "mushroom"
+
+#   main
+#   ------------------------------------------------------------------------
+
+if __name__ == '__main__':
+
+    # --- WIKI REQUEST ---
+
+    # get text from wikipedia
+    print('--- WIKI ---')
+    page = get_wikipage(wikipedia_page)
+    if not page:
+        sys.exit("--- STOP ---")
+    title = '<h1>'+page.title+'</h1>'
+    text = page.content
+
+    # print text in terminal
+    print('--- TXT ---')
+    print(text)
+
+    # --- APPLY TEXTRANK ---
+
+    # apply textrank
+    graph = similarity_graph(text)
+
+    # print ranked sentences in terminal
+    print('--- GRAPH ---')
+
+
+    # for i in len(graph.nodes()):
+    #     for j in len(graph.nodes()):
+
+    #         s1 = graph.nodes()[i]
+    #         s2 = graph.nodes()[j]
+    #         weight = graph.edge_weight((i, j))
+
+    #         print('---')
+    #         print('1. ' + s1)
+    #         print('2. ' + s1)
+    #         print('similarity: ' + weight)
+
diff --git a/summa/__init__.pyc b/summa/__init__.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..48d764ec602ab67848740ce5f34ab25bae1ab2f1
GIT binary patch
literal 339
zcmYjM&1%Ci3|9WL!UlsqL{8}ol(PGv2k+*K39(5{t0cq<UHW>v?rrt}n`CyFus%zc
zzE5)Y{kZ%3eCvDovy|^w_Hbmdd=j_<U;;V;OhKoB8R!f!2b}{JpbNkfbO~62t^jM$
zH6%8L_a<Wr|HIbQ98ty4Pl(ePsNi^5y7^2}y14B&EjG|ICv!Krr5i8i=KOFbXKBZ;
zVMXi~e|JmdI_oLWlN7d*x}JP**3naxBlkg~aB3d7A@)bQ;tW(GAm3E7pUMSf9aRd*
jk`4Pm2rrzr?LKPkjqyCkn0DUcumetZy}F~wk`R9Z>2*fl

literal 0
HcmV?d00001

diff --git a/summa/__pycache__/__init__.cpython-38.pyc b/summa/__pycache__/__init__.cpython-38.pyc
index f0224f0371da1eca43d7bacd531ea5bdb50fdaec..7801e26e7a13d4e61d7137908391da60d30875f4 100644
GIT binary patch
delta 107
zcmcb{^pS}-l$V!_0SL|<_D-&!$ScdJGf`VrIF}=egOMSHDVRZ%`6WnNlVf7M6&q`6
zN@hv%#2GS-oD;Wd2^Vn!WmYm2aRVtZ@hfrSTL*sq)QSSV{DQ>f%#uodJ-x|>jH&>2
CKpQ^*

delta 104
zcmey!bd8BOl$V!_0SG!zcqLa%<dtRAn5eDF&&ZI%6wIK>{1POk$v!dJN`|kvGOr{t
zxg;|=zBDhh<Q7LsYDGy=VqUhNCdb4BT6`-Ria-W|iC<w8|2nYf=I5kL_F+^70H<CZ
Aa{vGU

diff --git a/summa/__pycache__/edits.cpython-38.pyc b/summa/__pycache__/edits.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c80775cd3b7dc8b10f20465e6489ec274c0740c0
GIT binary patch
literal 1589
zcmb7^-EJF26vt=w!yjuqaic;K@gcV=5-TB_2Z$mhN^g)Xq)>z`U>WZW_BQ*)nK5aq
z%?<7g@CI<12jPw<m|L#;3ccc-N$i@6Trk%B_l)QBob#J`Iv7L@%U^$fnW{Epe^X<(
z_}KUitNsy1GRbpR@R`TS_Hw@nW}&nFyj8Sk9cKq<cW05a!@O7YXMJb4&>qYNXz$5L
z_FnSYp4^vxIY4V)cF$Nme8q8_nA^izEEI_HJh@QWVri62beh$d)RpE+l!;N#O)|es
zv?`4%Q>C%1H(zDBOco&OB{q*h71b{)S(Ttfx)k$VC1oYu&_kSPZbl|bF4XNlw8~K;
zByCrL(ut|w>|oofT;!QvKAENgo^w+$Zj9QWW7U9SSi=lod-}k5lCSx%-1rS^c;m1A
ztH1=(lm1KA1drJod#^$hHsP9cW?Hx3b`$>I`2%}paPGl(jR)T@*cp@I88clPbJa3Y
z6Ev+gW}*o&a;@p>@z)rVQ7u3e$|x8WD{V$|H4>wzWtO7fqiX(4rDpW8HV{8ZDb$<d
zp+TuW!>SKZ7U;tF(G_p_GxnSJGj4IsuDK0Ob*{>5&wAreZD$jEAcma0{$WxmnTd(4
zAQR=}u`put6;!mZo`_OpdOTN<o=?nDT?kOiYNauq5}Q9!z|BNGug6s_(#%{=#^Z@z
z6@{4KnnsW7OB<bF0!&&J_31r2E5*IVJoe&|OBm*D!^8L_uTqigQ#3YHUg2nFS=_Tx
zrn6ETf#g^p<6)z3M7~mA091ek+`*XbCp(Bii&i^M?!wIaxhiewkh9%ngESB?aoYxj
zu<hicT&$3G8|pgGjJ=QaWhSdiktf<z^+g3z+wRsc8#=!2{${;fRNo{4hIFps+X3m~
zzxrJi#v|VG`~CrH#~<=OYUIHq>`4iNO$pK1a(#l0JGu6dYk$kNf6H}%TnEIpBQ%%-
zo#6jscjHzMXTpGFi<<Ki2!c<r?t+jNSuOzixw+r}1w!0{`?T6SBoq@kBsn6Xti~aH
zK=y|uBa#Os4@v%g0*v!p`%xTqA7v-b9(;uD*oO{&o=juU8Pj-Z?`_k#dFg;IBfO7d
zk84mhRQS3ft&1@M<H%G1lqh#-+s4~aRHX*W)r||3IXI>jB6oivdx2BPF$bSZwL8&{
cGtMnXPUobk<SJLE^g`%8ls@17{OG-Z02<kp>i_@%

literal 0
HcmV?d00001

diff --git a/summa/commons.pyc b/summa/commons.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..812144379cf79e83cfbef39e5cb2969f2374229b
GIT binary patch
literal 865
zcmaJ<O>fgc5Pf5pswsp<DhI^5AYX8ixR$CCk&rm#;7C-h6h)4AiCz4Wc-Ii5^i)L4
zZ{lb54|HY{i4(PUpFO+t@#c-g`~Lpl_2(?Z_6hiX#@oGNFuVvQP!TQ(WCHvGE(4fQ
zKSG!tjH4&)S`b4d;KlsZri(liGu-Zv{B|E0Ja7YC6IO4r28NT@xDK!;ToKj*ZU}Cm
zCLgLv%K><uv&&d3j3P;z@P^-G$Wu418)MZ-jZs5=^u*a^XY0%=vAGcQI~-MbTSbyX
z!5#8f93Q)`s?y`Esj8-S@#3d{%ICALD9uzuCDQFPfO^!Y1A0N)rnD-J3p01Yuu5<T
zTxqkz$q;`Co^T7?627v6gx^q)0Cyx^4|(%qjlTtG`+65{?`YWvuX7&ASQ{H@Q5lvH
zdM`zf7+BBAE75jSniI2VT#fU7p(T$?;)Uu+QJb@7Zl^zNG0%M@3B5r4PSt9BZd*M~
zq>kz)N<S)@>#^&}+}3urXiuNB;UwiQ3PwblNTiZP&Z}Tu<t3;V7PF>xIyYm>%_N1e
z*4MKC=p-_S{|@dYU$(YtzT0V6w>Hi4bXHo;=$=X7E(7S0Vj3waEunu^B3H=)!=|D*
XKH9SKJ*gVgmG)HVI_Yu{_VykC#c-9n

literal 0
HcmV?d00001

diff --git a/summa/edits.py b/summa/edits.py
index ebb7b5d..b77fb96 100644
--- a/summa/edits.py
+++ b/summa/edits.py
@@ -30,4 +30,18 @@ def scored_sentences(text, language="english", split=False, additional_stopwords
     # Adds the summa scores to the sentence objects.
     _add_scores_to_sentences(sentences, pagerank_scores)
 
-    return sentences
\ No newline at end of file
+    return sentences
+
+
+def similarity_graph(text, language="english", additional_stopwords=None):
+    if not isinstance(text, str):
+        raise ValueError("Text parameter must be a Unicode object (str)!")
+
+    # Gets a list of processed sentences.
+    sentences = _clean_text_by_sentences(text, language, additional_stopwords)
+
+    # Creates the graph and calculates the similarity coefficient for every pair of nodes.
+    graph = _build_graph([sentence.token for sentence in sentences])
+    _set_graph_edge_weights(graph)
+
+    return graph
\ No newline at end of file
diff --git a/wikipage/__pycache__/__init__.cpython-38.pyc b/wikipage/__pycache__/__init__.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1c40f123a2981f744f91e6495036cb8feace50b2
GIT binary patch
literal 228
zcmYj~F$%&!5Jh)miXaHq-XP7+1w<_Dgdm8GO&GJIu9|Gv-Jl-Hn|Ow_R@Qb_P6WXZ
z|9{@V@K=LDU$C<KyclsmI`~(_FlGB8hZb6%L@C!&3LPy)+I#X<q_T}`+)<WBZE`k8
z(}C+gZ^Fe~-*{^%NgD)hoxuub7!VfD9;gT&utMY2hFo!ku%|OR+S|qxR+@2Z9r*xs
jsg<p<FyRVH!EXxk+*BsyxvH;iGA^xdPBf7`{9=m_Ot3vR

literal 0
HcmV?d00001

diff --git a/wikipage/__pycache__/page.cpython-38.pyc b/wikipage/__pycache__/page.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6e1388abcb66d02d9d74a70334a8bdb323b50710
GIT binary patch
literal 679
zcmYjN!EO^V5S{UElFcfCP!NJk@3~}8+z>({AXP|FrGf%`0e5XSyGgt&+pDHk(gScp
zs>CO>N4|n9XO6yd>Mw9&oKUG_dF+{a&u_+W`~4Wme*Su${bm4vd~n+W3ZBz8`{V=&
zTtEsEE>W;6L=b@pze36c9>K76O&FLsCRK+1`rkr2y~+#P<~unHYxqL!vx&u$t?5<4
zL~s#o@T|8+5k7_uOhN63V1s$A#|VB|ybRZHg`e>~DE5xxvPIk$t@cGQg(Dz5mhG}b
zoNovlMhbhR#mdO5iOz;lPiTq_J173`m32U!N0V#c@T(zn9W70>uqKU7ljqXdqEgxy
z-%v5qQWRQdHtp0}E-Ee@47;vX>q1%AJt^izO$hGVMsl6aTzsH}T)ot~(rNdQ=W=)L
zPj8Su=VH0aWbKtqdVgFPzC14ShWo@8?>g14It<gOCWhimsr|P)lrHp{AE572{P$zB
z^bI7U(gjz^m|LD4>Z+ukO~Mr~%;;F^Y@XPeJmFf-s)kxsg2GP~jcyaUsz+7Lv%;Px
zqfxS*i3Ad_GOABqFPHYiO*SG;^N1V-5i%Um?%^Faz&(6`F+ODa9@U0{_P{hUL|t%|
cZ?oHmZ@B?MT=aBViDn_6`7bwq#E8YMe{2`69{>OV

literal 0
HcmV?d00001

diff --git a/wikipage/page.py b/wikipage/page.py
new file mode 100644
index 0000000..9dc510f
--- /dev/null
+++ b/wikipage/page.py
@@ -0,0 +1,25 @@
+import wikipedia
+
+wikipedia.set_lang("en")
+
+#   wikipedia
+#   ------------------------------------------------------------------------
+
+def get_wikipage(pagename):
+    # get wikipedia page content by name of the page
+
+    print(pagename)
+    try:
+        results = wikipedia.search(pagename, results=1, suggestion=False)
+        try:
+            pagename = results[0]
+        except IndexError:
+            # if there is no suggestion or search results, the page doesn't exist
+            raise wikipedia.PageError(pagename)
+        return wikipedia.WikipediaPage(pagename, redirect=True, preload=True)
+    except wikipedia.exceptions.DisambiguationError as e:
+        print(e.options)
+        page = ''
+
+    return page
+