From 8706e225f2db60a2eb82fe38f8f1a855cec8625b Mon Sep 17 00:00:00 2001 From: Ksan Date: Mon, 7 Jul 2025 23:26:35 +0200 Subject: [PATCH] moving to laptop --- build.gradle.kts | 2 +- src/main/java/dev/ksan/DataStore.java | 32 +++++++- src/main/java/dev/ksan/DataThread.java | 102 +++++++++++++++++++++---- src/main/java/dev/ksan/ETFScraper.java | 16 ++-- src/main/java/dev/ksan/Main.java | 10 ++- src/main/java/dev/ksan/User.java | 3 +- 6 files changed, 138 insertions(+), 27 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index e9a0f78..a907d12 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -15,7 +15,7 @@ application{ } dependencies { implementation("org.simplejavamail:simple-java-mail:8.7.0") - implementation("net.sourceforge.htmlunit:htmlunit:2.53.0") + implementation("net.sourceforge.htmlunit:htmlunit:2.70.0") testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") } diff --git a/src/main/java/dev/ksan/DataStore.java b/src/main/java/dev/ksan/DataStore.java index 8aeeb0a..5275819 100644 --- a/src/main/java/dev/ksan/DataStore.java +++ b/src/main/java/dev/ksan/DataStore.java @@ -4,7 +4,7 @@ import java.io.*; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -public class DataStore { +public class DataStore implements AutoCloseable { public static Map subjectSubscriptions = new ConcurrentHashMap<>(); private static final String DATA_FILE = "subjects_data.ser"; @@ -12,9 +12,24 @@ public class DataStore { loadSubjectsFromFile(); } - public static void notifySubject(Subject subject, SubjectEntry entry){ - subjectSubscriptions.get(subject).notifyUsers(entry); + public static void notifySubject(Subject subject, SubjectEntry entry) { + if (subject != null && entry != null) { + Object subscription = subjectSubscriptions.get(subject); + if (subscription != null) { + ((Subscription) subscription).notifyUsers(entry); + System.out.println("Subject " + subject + " has notified " + entry); + } else { + System.out.println("Warning: No subscription found for subject " + subject); + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } } + public static synchronized void subscribeUserToSubject(User user, Subject subject) { if (user != null && subject != null && subjectSubscriptions.containsKey(subject)) { subjectSubscriptions.get(subject).subscribe(user); @@ -33,6 +48,13 @@ public class DataStore { private static void loadSubjectsFromFile() { try (ObjectInputStream in = new ObjectInputStream(new FileInputStream(DATA_FILE))) { subjectSubscriptions = (Map) in.readObject(); + for (Map.Entry entry : subjectSubscriptions.entrySet()) { + System.out.println("Subject " + entry.getKey() + " has notified " + entry.getValue()); + } + System.out.println("Loaded subjects from " + DATA_FILE); + try{ + Thread.sleep(1000); + }catch (InterruptedException e){} } catch (IOException | ClassNotFoundException e) { System.out.println("Error loading subjects data: " + e.getMessage()); } @@ -46,4 +68,8 @@ public class DataStore { } } + @Override + public void close() throws Exception { + saveSubjectsToFile(); + } } diff --git a/src/main/java/dev/ksan/DataThread.java b/src/main/java/dev/ksan/DataThread.java index 1aaf8e2..94d7844 100644 --- a/src/main/java/dev/ksan/DataThread.java +++ b/src/main/java/dev/ksan/DataThread.java @@ -4,32 +4,108 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.io.*; +import java.util.*; + public class DataThread implements Runnable { private List allEntries = new ArrayList<>(); private boolean running; - Set subjects = Subject.generateSubjects(); + private Set subjects = Subject.generateSubjects(); + + private static final String FILE_NAME = "allEntries.dat"; + @Override public void run() { - while(running){ + running = true; + try { + readEntriesFromFile(); + }catch (Exception e) { + e.printStackTrace(); } - } - public void compare(List subjectEntries) { - for(SubjectEntry subjectEntry : subjectEntries){ - if(!allEntries.contains(subjectEntry)){ - for(Subject subject : subjects){ - if(subject.matchesTitle(subjectEntry.getTitle())){ - DataStore.notifySubject(subject, subjectEntry); - allEntries.add(subjectEntry); - } - } + while (running) { + try { + System.out.println("comparing: " + compare(ETFScraper.getEntries())); + Thread.sleep(20000); + } catch (InterruptedException e) { + System.out.println("Thread interrupted, stopping..."); + running = false; + Thread.currentThread().interrupt(); + } catch (Exception e) { + e.printStackTrace(); } } + saveEntriesToFile(); } - public void stop(){ + public boolean compare(List subjectEntries) { + if (subjectEntries.size() == 0) { + System.out.println("No entries found"); + return false; + } + + for (SubjectEntry subjectEntry : subjectEntries) { + // If entry is new and not already present + if (!allEntries.contains(subjectEntry)) { + for (Subject subject : subjects) { + if (subject.matchesTitle(subjectEntry.getTitle())) { + DataStore.notifySubject(subject, subjectEntry); + allEntries.add(subjectEntry); + break; + } + } + } + } + return true; + } + + public void stop() { running = false; } + + private void saveEntriesToFile() { + try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(FILE_NAME))) { + oos.writeObject(allEntries); + + System.out.println("Entries saved to file."); + } catch (IOException e) { + e.printStackTrace(); + } + } + + private void readEntriesFromFile() throws IOException { + File file = new File(FILE_NAME); + + // Check if the file exists before reading + if (!file.exists()) { + try { + if (file.createNewFile()) { + System.out.println("No saved entries found. Created new file."); + } + } catch (IOException e) { + System.out.println("Error creating file: " + e.getMessage()); + e.printStackTrace(); + return; // Return early if file creation fails + } + } + + // Check if file is empty BEFORE reading + if (file.length() == 0) { + System.out.println("File is empty. No entries loaded."); + return; + } + + // Now try to read from the file + try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(file))) { + allEntries = (List) ois.readObject(); + System.out.println("Entries loaded from file."); + } catch (FileNotFoundException e) { + System.out.println("File not found, but it was already handled above."); + } catch (IOException | ClassNotFoundException e) { + System.out.println("Error while reading the file: " + e.getMessage()); + e.printStackTrace(); + } + } } diff --git a/src/main/java/dev/ksan/ETFScraper.java b/src/main/java/dev/ksan/ETFScraper.java index 3ec39ab..bd274a7 100644 --- a/src/main/java/dev/ksan/ETFScraper.java +++ b/src/main/java/dev/ksan/ETFScraper.java @@ -11,7 +11,7 @@ import java.util.ArrayList; import java.util.List; public class ETFScraper implements Runnable { - private List entries = new ArrayList<>(); + private static List entries = new ArrayList<>(); private volatile boolean running = true; @@ -21,14 +21,16 @@ public class ETFScraper implements Runnable { HtmlElement element = parent.getFirstByXPath(xPath); return element == null ? "" : element.asNormalizedText(); } - List getAllEntries() { - return entries; + static List getEntries() { + synchronized(entries) { + return new ArrayList<>(entries); + } } @Override public void run() { while(running) { - try (final WebClient webClient = new WebClient(BrowserVersion.FIREFOX)) { + try (final WebClient webClient = new WebClient(BrowserVersion.CHROME)) { webClient.getOptions().setJavaScriptEnabled(true); webClient.getOptions().setCssEnabled(false); @@ -47,7 +49,7 @@ public class ETFScraper implements Runnable { int ul_idSelection = 1; for (HtmlAnchor anchor : toggles) { String groupName = anchor.asNormalizedText().split("\n")[0].trim(); - System.out.println("Group name: " + groupName); + //System.out.println("Group name: " + groupName); HtmlPage updatedPage = anchor.click(); webClient.waitForBackgroundJavaScript(1000); @@ -57,7 +59,7 @@ public class ETFScraper implements Runnable { HtmlElement listElement = rawElement instanceof HtmlElement ? (HtmlElement) rawElement : null; if (listElement == null) { - System.out.println("An element with id " + ul_id + " was not found"); + //System.out.println("An element with id " + ul_id + " was not found"); ul_idSelection++; continue; } @@ -82,7 +84,7 @@ public class ETFScraper implements Runnable { ul_idSelection++; } - Thread.sleep(20000); + //Thread.sleep(20000); } catch (Exception e) { e.printStackTrace(); diff --git a/src/main/java/dev/ksan/Main.java b/src/main/java/dev/ksan/Main.java index 38e0332..1ac1ea3 100644 --- a/src/main/java/dev/ksan/Main.java +++ b/src/main/java/dev/ksan/Main.java @@ -31,8 +31,11 @@ public class Main { Thread webClientThread = new Thread(task, "WebClientThread"); webClientThread.start(); + DataThread data = new DataThread(); + Thread dataThread = new Thread(data, "DataThread"); Scanner scanner = new Scanner(System.in); + dataThread.start(); try { while (running) { @@ -40,7 +43,10 @@ public class Main { switch (command) { case "stop": + dataThread.interrupt(); + data.stop(); task.stop(); + webClientThread.interrupt(); running = false; System.out.println("Stopping..."); break; @@ -65,11 +71,11 @@ public class Main { throw new RuntimeException(e); } - entries=task.getAllEntries(); + entries=task.getEntries(); System.out.println("BBBBBBBBBBBBb"); System.out.println(entries.size()); - for(SubjectEntry e : task.getAllEntries()) { + for(SubjectEntry e : task.getEntries()) { //System.out.println(e); } for(Subject subject : user.getSubjectSet()) { diff --git a/src/main/java/dev/ksan/User.java b/src/main/java/dev/ksan/User.java index 3f586cb..42f198a 100644 --- a/src/main/java/dev/ksan/User.java +++ b/src/main/java/dev/ksan/User.java @@ -56,7 +56,8 @@ public class User implements Serializable { public void sendNotification(SubjectEntry entry){ if(notificationMethod == NotificationMethod.EMAIL){ - sendEmail(entry); + System.out.println("Sending an e-mail notification"); + // sendEmail(entry); } else if (notificationMethod == NotificationMethod.PUSH_NOTIFICATION) { pushNotification(entry); }