From a645184cff1f62b92280fe4a05ae3983b07e80c0 Mon Sep 17 00:00:00 2001
From: Dennis Eckerskorn <denniseckerskorn@gmail.com>
Date: Sun, 8 Dec 2024 17:40:46 +0100
Subject: [PATCH] Added scrapper, still fixing issues, just a save in between

---
 .../threads_manager.cpython-313.pyc           | Bin 9280 -> 9582 bytes
 src/services/scrapper.py                      |  89 ++++++++++++++++++
 src/services/threads_manager.py               |   4 +
 .../centered_window.cpython-313.pyc           | Bin 11663 -> 12826 bytes
 src/ui/centered_window.py                     |  16 +++-
 5 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 src/services/scrapper.py

diff --git a/src/services/__pycache__/threads_manager.cpython-313.pyc b/src/services/__pycache__/threads_manager.cpython-313.pyc
index 1de9aa91ca936380da24beda6a2144569e3a734b..394c7df1846465601e4a244c6cdfdb43b343c262 100644
GIT binary patch
delta 1833
zcmZWqU2Icj7(U<G`D;&4x30fyyRO}TK{gbZ@fXIx#SN4Q-AsmHSH@Xw*xKehX9K4Z
zh>?s$1Iz1;UZI$X#zf5$Z<NFfuS_6WB%lWfCd!3g5H?M`BYy99x`DCDo__nz`}@4l
zd;0zGgMI$L48tq1kM-kN@oH$%A0w|Vc8#>>gq*zJyHBTjT8Ig)LQaVbIkitjQ$e7<
zBFSot3w^#UakShdEmW%}UA%gh=rBN<A?fjuZ(xcP;ad_WGAxsL*DJc~D+ptxkGSVC
zVOm+cEecujKvsrT((Q9Js}?h>LQ)(EQ8c3s46>plS?2T<4S*@mD+ar=hT*>0)qW@1
z25+tCO2oS$z3YAG7dpBYq#Ii1PAq*X@l$Nmg7lsNEp<0{G%_DYoiS}%nzS70g!#Hf
z>o9%@<JZwJp1F}yGs36obecwBv$UNg;e@o4G{FPCN&H-duj}Jbj;QdvbQ4}RLU3J4
z5-+UCL3qgi4A~%lxEM>QqpWk`ke@&4XbOslg0SL=LyMx55EL3>@S7AR5q1heMF|>B
zEGPdkr#J+BLB``{Wh29TQi9d6;t9ed#VadCg^^^a&o=v*IYmOK9|75-^$ZSV>uC_i
z)q#xTx9kG7^PJsMxyZYu2{cS1gy0jkZBu|9oyOVf@q%sUZKTP{7Yj3HR&=RpX3s5D
z0%zoJb+girH;-V0b|mw&=DXlM@4dKvSxep0+LyKVOQWA2{pjeej=|-Q!5i8hIH$cs
znwET?o1{00ndFjf&agRHcBSgbRaz?Bd=l;mR61KOvr=&`5YS_cp$&Qu>4$OseHP|{
z-YJrL2>W}B7>xEJU<15!$9J++o~q={Qzgr`9KF^PHbapuN4BO-v+8(;PEDFsyHqLD
z9(G-8)FI}2n;5YhJu+}Q8r2_@GPHEf|DE_UO2<vkg=5q*x%%O`Xg`{8uD9ZeC#s1J
z8Kv78SnEo|n@+eF=q3l3w8kGq;)83UD<lAO;in8#C@<~I8K%iE!#l_X+>69Pjx;k}
zPN(PKgGfNxgE5|iuOgwue;n|77-57#fjRa!qS*exrObM$%OiA0oBOyD<ieC~(#Jil
zZO984;&}$V=!xCCX#{UtVLsYIj=+b}FWenjnuvW&hyiu+D{XE`n5U*CzQ3p?>;&D1
z;d*>4seqaoV(u>{dX!o{XA|)pAM8OiTc_}ex@0a-M;&}=B)@-X+>!Q<jE-|!`Bqpm
zbDUsnTr0GV{p*es{)~Ic)a8bxNX8+M>el#d6ehUA)E0%8br|MT-Q*-(PE~7NZED#@
z*k%{gZ|=f%n{V7(q13EcX{%IDGgZ;;Y5GLFjyzeJWy=2lBp{R7Le9fjrhk-=fxm^c
z+8D4#g%--kD>IIY-?$x*Y2Yi5!dm0A!p>Y76SXee;nz%tybWUOToS`)W%tE^GIiIC
z^BaH*twX1yc$q@TB0PhDs;8X@tq8~(J&b@$?}YPt_DZo!llg+pUKGb>ZPUu<KNBeC
zawn?c2?2iqEF9c9;TI{qD(Z62YD$s^RvYT&A+p-3L#{2ui{$26gC0Ov-s3SmQMlMP
zPMv^d(z(*4Y4xoAO1M7{Jkc9yxSrh6cFt6e&rX}W=o$6^6>ec`59<gaw}txKg3i&n
aFZA9Op1vz=yeoA6BkdMRVpTxoq5cJG3Ze)A

delta 1621
zcmZux+fN)-7@srDWv?u|%U)P^-5V4ptstQY7Ag?BO|cQR&>|5tY?ooPDLYHf%u;Nu
ztx2mq7!CP&H)&rm@x_q7X`<0TKp)z~sV4TZ4{a<yX!OPNo!u^&I1j%$^PTfuf4?(N
zU;Fl4`yYP4M_`Y2Z=w8mc&9y1mUsGQ<8dJ;%*zR3UKtT_TM($HOf+?8cVy@Ohr|Pm
zBn3Xdo7{pANLiXE0}}j05^7n{#5qlZ39(<)WY`cVJ(}o9ma)D@1Mt0gP4Tf<)+o%h
z^@X3g<)~*{`o#0QPw4I2mhQW<@M+tF@eX#xQLBbcD<#X3t{CrGv<*uLG0@s*7}xxe
z(maH_nM{Uu!spT{(glt*A;p*tehf#+Ss0T4f-~NvxFEokHwM*Ur+6U%Z+TM62|+N)
zMfgsMK!=(lAsA5m{1N8KpNqnUDEyQO6A3oe<8WF@k~nh<!*$uMl;yIlDe#+mGVW!k
zDK<v^2<;5!G(Qc(kZaOMyU><E2*Hjk|6+hGolt#awP+i<ja^!LxmYzcvC&yK+qZD)
zyrD;~my|48eubEO8Ft;bNo+sic|b@Io_XU@oJnICDy^Gl#Vq55Anl8jP8jnIkYOnM
zc3acfkNEu}X@`7ZupC5JHUWWRNxW>wceP@!)pTR4V%e7CZB~ipQS8!@t#!lLaNLEh
zlCfb|Y9>9-%+1mrVzdg!h`s2Mg-6|SPpix6{fB|C#Y?<wo|_ZtZzgK+$y3pj7=jOP
z1XnyylYATzeVKvPVjeAdcoOO-ukT00KZwK&4<mad0Jowi{HPdS+0C0wBi}^F$uc~N
zB_JG2kr)icJ|3Aw|7iwHuD(*8Qp2=$WQcRp^2JjHgtH9ZWidu8sI$j}f##;Ii8?Oy
zwJTM_X)|kA9hYU(HT>kGcI4t3i+tGTb5YM?=ras>Un^5nv=c2E22O&_F($DPZ^K^v
z%R@rVt~-SIAlSW^Khi`)N~p>I)(*dPkC7`7Nfub+?<5D5M#=D5GBM9pFoy|S13yxt
z%tJcr^qaH#`NEPT&CD(=aT2)nEtxs4uuIsi88yqf&yve`Ur34M5~S1pF3zID1o}pL
zSmDhr!gji!SnySP<FKjm%qv8}%k)@e^lVP@i*+qDQ)9%`tc+DLGfao{RE8eu*^x`N
zx@rFpV=$N-CbyxK8(rXZVPeg7gf4{^&DC1fQSn`2F%D;evgcZSem~clfe&@1*=i1+
z=Cb4lbmTW<7+#b3h2+1hMveg7$ro<me}yIyjv)*p;1!{HgbV^sjV>bO7&sAKx9p-_
zDd|O<-Jr{L+pu)~3xQ&qhBj1OA>eP^8e{O0@Us*-5Z!X$K|+!z4?5cA0y*gM!dg!@
z$sO^y=B4M^t3j}is~X(xS<-kcx`l>T62~=Bt*zGAjaTXWYy-t^9fQ-oso<Z|bwaui
I1Vn!RUyh7r8vp<R

diff --git a/src/services/scrapper.py b/src/services/scrapper.py
new file mode 100644
index 0000000..6ab0456
--- /dev/null
+++ b/src/services/scrapper.py
@@ -0,0 +1,89 @@
+import threading
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin
+import mysql.connector
+
+class Scrapper:
+    def __init__(self, ui_instance):
+        self.ui_instance = ui_instance
+        self.visited_links = set()
+        self.running=False
+        self.lock() = threading.Lock()
+
+
+    #Configurar la base de datos para los enlaces
+    self.db.config = {
+        "host": "localhost",
+        "user": "root",
+        "password": "1234Scrap",
+        "database": "scrap_links_db"
+    }
+
+    def start_scraping(self):
+        """Inicia el proceso de scraping"""
+        self.running = True
+        url = self.get_url_from_ui()
+        if url:
+            self.scrape_page(url)
+
+    def stop_scraping(self):
+        """Detiene el proceso de scraping"""
+        self.running = False
+
+    def scrape_page(self, url):
+        """Scrapea una web y busca los enlaces"""
+        if not self.running or url in self.visited_links:
+            return
+        
+        with self.lock:
+            self.visited_links.add(url)
+
+        try:
+            response = requests.get(url, timeout=10)
+            if response.status_code == 200:
+                soup = BeautifulSoup(response.text, "html.parser")
+                links = [urljoin(url, a.get("href")) for a in soup.find_all("a", href=True)]
+                self.update_ui(url, links)
+                self.save_links_to_db(url, links)
+
+                for link in links:
+                    if self.running:
+                        threading.Thread(target=self.scrape_page, args=(link,), daemon=True).start()
+            else:
+                print(f"Error al acceder a {url}: {response.status_code}")
+        except Exception as e:
+            print(f"Error al scrapear {url}: {e}")
+
+    def update_ui(self, url, links):
+        """Actualiza la pestaña 'Scrapping' con los enlaces encontrados"""
+        tab = self.ui_instance.tabs["Scrapping"]
+        text_widget = tab.text_widget
+        text_widget.insert("end", f"Enlaces encontrados en {url}:\n")
+        for link in links:
+            text_widget.insert("end", f" - {link}\n")
+            text_widget.insert("end", "\n")
+            text_widget.see("end")
+
+    def save_links_to_db(self, url, links)
+        """Guarda los enlaces en la base de datos"""
+        try:
+            connection = mysql.connector(**self.db_config)
+            cursor = connection.cursor()
+            cursor.execute("CREATE TABLE IF NOT EXISTS links (id INT AUTO_INCREMENT PRIMARY KEY, url TEXT, parent_url TEXT)")
+            for link in links:
+                cursor.execute("INSERT INTO links (url, parent_url) VALUES (%s, %s)", (link, url))
+                connection.commit()
+                cursor.close()
+        except:
+            print(f"Error al gaurdar en la base de datos")
+
+    def get_url_from_ui(self):
+        """Obtiene la URL desde la interfaz de usuario"""
+        try:
+            url_entry = self.ui_instance.left_panel.url_entry
+            return url_entry.get()
+        except AttributeError:
+            print("No se pudo obtener la URL desde la interfaz")
+            return None
+    
\ No newline at end of file
diff --git a/src/services/threads_manager.py b/src/services/threads_manager.py
index 4c13bec..1e0fed9 100644
--- a/src/services/threads_manager.py
+++ b/src/services/threads_manager.py
@@ -6,6 +6,7 @@ import random
 from services.threaden_task import ThreadenTask
 from services.system_monitor import SystemMonitor
 from services.tetris_game import TetrisGame
+from services.scrapper import Scrapper
 
 class ThreadsManager:
     """Constructor"""
@@ -17,8 +18,10 @@ class ThreadsManager:
             "temperature": ThreadenTask(), 
             "emails":ThreadenTask(),
             "tetris_game":ThreadenTask(),
+            "scrapper":ThreadenTask(),
         }
         self.system_monitor_tasks = {}
+        self.scrapper = Scrapper(ui_instance)
       
 
 
@@ -35,6 +38,7 @@ class ThreadsManager:
         self.tasks["time"].start(self.update_time)
         self.tasks["temperature"].start(self.update_temperature)
         self.tasks["emails"].start(self.update_emails)
+        self.tasks["scrapper"].start(self.scrapper.start_scraping)
 
         if self.system_monitor:
             for metric in self.system_monitor.metrics.keys():
diff --git a/src/ui/__pycache__/centered_window.cpython-313.pyc b/src/ui/__pycache__/centered_window.cpython-313.pyc
index 0724c488a80a239a5f4f78330dad1e0d1642abac..8b00711da49c33b52191c0b199226e62f8471ac2 100644
GIT binary patch
delta 1487
zcmY*YZ){Ul6o0o}$J=Yyu4}jT&(^)cI(TeTMjT-v2+RpvCeVNKPwl$(F}l6Jj@vh6
zm4O1$4wejiqC-CDNC*Z#@F$JQ)Grb>k?4oMh`!|skq<^c7!n5<`J(aOHX`1%zkANP
zzkAO4y_`F<?_Avapt6!<$hvYfGW>S)b?ZIO@PRS^7xx1KWZ~He3e|+HAsgJO^}^>4
z4*D$(P}Q`oqBd05WstcD2dkDkA!;$TJG%@#6G34{Dkh{!BV3pXI;r*lC7EHf(*X~7
zNA*&2CFF`Q&!CXo!;G*Y4>Y;FFkmSwqY2?xzL6_(yd;$i0;~gh;6{zZP;Y|2Yh6n`
zT}IgwvCuB6LJgg$RETGW8Nrv=EMJ%H(<?$lI5swxkcTr>y|RK+@$n&%mtuS**e@Wx
zYD|g^i3d_rT*OgDJgj8wy>enG5ySlBi$hQLZzPU`{2!bF+3Evxi=#Y*Oe8>?Kw+@^
zYT%~B2_r4e=l*S|nGp=zbv9k>M8tv#SCb2P9Rb~xR$^Gew1#di+R-A9G@P?}j>Snl
z-AUll8_a@V(EMvv|60wzQT1<}&#3-B&A(Ii?<@o&S|F|l;#wf729jFfh#EL@%Rh-%
zkeinbcsJSbs{}Ik=S4-7#V6DiOJ{6jvG~M>&NiOxhc|Sr5^TD0NEwBT_O;LIY<GCH
zhkUyUos-1<N^~qHi;`Y3j-{wb!~e;_>q!xz64%p$2MNHUecgK9R+3IMMWxuFC}B*F
zbW2I@;JBisWZj%rQe)AQlddG^13X#KQQc+u1c6lj+p$G6)70^>n(_KRY&+LBmzg&g
z`l705AZuG->om4rW$Q0=f7Eln=Ng(f6sW)cwzF=wKi5C!DQw)SIs@5?1=c=eowA<Y
zJhvrlRoUftJ<Zv&Y1=|`d$w%GHf6iTHr=lC%uePe=k^xX^gd)z7#&56400U{pseLC
z>(p4c%DT@UoWlibt^B#xJ)6uW=lTkr+X}&#w4kU4MJ+h021g6hAvO4#7CfSoU+rYp
zyudm%)}^wpf_vrnY-|2y#}<=$l4vC(F+eu>)aBoFoEUg74VK{^YEuG!fx=b_ZxRqt
zF-kTX%5Wh%MEurRQpEeA#occnrpyBr5-{oRMk%=A_M#EE<6eW3;PmV<;R$jIuY2Oe
zze_Nf@A2l4A&B5!gBKpwUqn-|soDuw8|>&j+-Nvk`#w##nn0R9G#<B5|8$Xb!e>X9
z;9cKmXc_{I%hA>RMB{E_<59v8j3pJ11?c5H@*(oTNT<Iq{ju|jP6=B#1$(z{3*%9m
zJ4r#NQI1Stn7q!x)L|X%#;IKMCG>Ira`P1<ItqJM_{=9L^)iJEaD2t79gnl>++d2#
zS28+CDkf;I@+~{U;T<pHYm`k(=;nmHKUJE{v|fHFCXI_}JVH(cLur!VgMGpp^j$tD
z>_zB0+*`?`U%=Yxu3Un@pjp16a1fquZAah2E3IMA6b+oFFiqhp3acnwhi_W@e4B~+
z=q=_?gV)q`e;sRTyYJ#mb@%I+5o<ew)otB2DnMVNj3#77#P}4PZ2J=Z1ikGYHG7Om
RI1`%P_0Fz87-|<E`9Iq^uoeIS

delta 700
zcmXAmPiPZS5XSS;CQUX?N@>$d(xgqJCMzaQiWC$CG1&N*Kod!=#AK7*q@l^CeY<J3
zhngO|h*0euD)nIXVChAW)k7&>ih_9ZV1uk8-aLqc6_1K<;vRnQd*6IBZ(!HOm8*_l
zb~|TaANL|vn)W?&eCLc!qW$5Tq`hR|o!Udkn|32_1+VjD4{xOdEUI%Asi<aW1zD=V
zn)8HNu`)LYoqflRKMWA>`)1m=8GKQ29dv7+0|C>(9INxD#oh@?saI4%tSLju0u^Rw
z%W?@4o<s3@7NAp@YNuyVV+S2a#1VA{p0r$cptZ;)Re`EQaiJ=08R;aZuEBH9MOzM~
zfXIW*8za+@@VZF>X1rsh2#>v29D1jMz1e(SRcrF<ph&VHF=n*4gR6vOAmiI5OYk$)
z3&)1L3BdU9{k|2PILe?9h9P<w?~PW-;_oI8z~$c|%h2=($Rq8qf3l5GNC!M*0qTK|
zz8t3iFU6E@nm89fo1!ARMMMeaf<0{2Tfr@|sg3YYvHkN=ukAWkHxTR45xSkop~OKt
zm#?w)tJ%CjX&w#p=|n1(xI~{|1gGk@vRtfXD?(nXD7txGsMIBe7FdNhw*Dl&17AX8
z<h9lxo+7bLY;ngVS(T_HrpvNeTL_EwYV}4|D5&L{{DB1&8$-kJ$zp%!2<%ycG>GHO
zbekP?2ewB-WEZ}Uq+CmAG!V;(Fd~B3gv*ih<P>Z~`b|#Lk^ND48Hsh}SV`|tXUh0G
ey$M~>S7Z;~N2AUQZG^v@Tua|e|2Cj*z4#9Tmc`Wo

diff --git a/src/ui/centered_window.py b/src/ui/centered_window.py
index 1bb970d..567df02 100644
--- a/src/ui/centered_window.py
+++ b/src/ui/centered_window.py
@@ -90,6 +90,20 @@ class CenteredWindow(ctk.CTk):
                 btn = ctk.CTkButton(left_panel, text=text, command=command, width=150)
                 btn.pack(pady=5, padx=10)
 
+        scrapping_label = ctk.CTkLabel(left_panel, text="Scrapping", font=("Arial", 12, "bold"))
+        scrapping_label.pack(anchor=ctk.W, pady=5, padx=10)
+        url_entry = ctk.CTkEntry(left_panel, placeholder_text="Introduce la URL")
+        url_entry.pack(pady=5, padx=10)
+
+        self.left_panel = left_panel
+        self.left_panel.url_entry = url_entry
+        start_button = ctk.CTkButton(left_panel, text="Iniciar Scrapping", command=lambda:
+                                     self.thread_manager.tasks["scrapper"].start(self.thread_manager.scrapper.start_scraping))
+        start_button.pack(pady=5, padx=10)
+
+        stop_button = ctk.CTkButton(left_panel, text="Detener Scrapping", command=self.thread_manager.tasks["scrapper"].stop)
+        stop_button.pack("pady=5, padx=10")
+
 
 
     def create_center_panel(self):
@@ -101,7 +115,7 @@ class CenteredWindow(ctk.CTk):
         tab_view.pack(fill=ctk.BOTH, expand=True)
 
         # Crear pestañas y manejar contenido por separado
-        for tab_name in ["Resultados Scrapping", "Navegador", "Correos", "Juego", "Sistema"]:
+        for tab_name in ["Scrapping", "Navegador", "Correos", "Juego", "Sistema"]:
             tab = tab_view.add(tab_name)
 
             if tab_name == "Sistema":