fixed buggs and added state saving

This commit is contained in:
Ksan 2025-07-10 16:41:54 +02:00
parent 8706e225f2
commit 20c5450631
10 changed files with 338 additions and 119 deletions

2
.gitignore vendored
View File

@ -1,4 +1,6 @@
.gradle
*.ser
*.json
.idea/
# Ignore Gradle wrapper files (if you already have gradlew and gradlew.bat)
gradle/wrapper/

View File

@ -18,6 +18,7 @@ dependencies {
implementation("net.sourceforge.htmlunit:htmlunit:2.70.0")
testImplementation(platform("org.junit:junit-bom:5.10.0"))
testImplementation("org.junit.jupiter:junit-jupiter")
implementation("com.fasterxml.jackson.core:jackson-databind:2.15.2")
}
tasks.test {

View File

@ -3,15 +3,44 @@ package dev.ksan;
import java.io.*;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
public class DataStore implements AutoCloseable {
public class DataStore{
public static Map<Subject,Subscription> subjectSubscriptions = new ConcurrentHashMap<>();
private static final String DATA_FILE = "subjects_data.ser";
static{
private static final long SAVE_INTERVAL_MS = 10000;
private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
private static volatile boolean running = true;
static {
loadSubjectsFromFile();
executorService.scheduleAtFixedRate(DataStore::saveSubjectsToFile, 0, SAVE_INTERVAL_MS, TimeUnit.MILLISECONDS);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
running = false;
saveSubjectsToFile();
executorService.shutdown();
}));
}
public static void stop(){
running = false;
executorService.shutdownNow();
try {
if(!executorService.awaitTermination(3000, TimeUnit.MILLISECONDS)){
executorService.shutdownNow();
}
} catch (InterruptedException e) {
executorService.shutdownNow();
throw new RuntimeException(e);
}finally {
saveSubjectsToFile();
}
}
public static void notifySubject(Subject subject, SubjectEntry entry) {
if (subject != null && entry != null) {
Object subscription = subjectSubscriptions.get(subject);
@ -31,7 +60,15 @@ public class DataStore implements AutoCloseable {
}
public static synchronized void subscribeUserToSubject(User user, Subject subject) {
if (user != null && subject != null && subjectSubscriptions.containsKey(subject)) {
if (user != null && subject != null) {
if(!subjectSubscriptions.containsKey(subject)){
Subscription subscription = new Subscription(subject);
subscription.subscribe(user);
subjectSubscriptions.put(subject,subscription);
user.addSubject(subject);
return;
}
subjectSubscriptions.get(subject).subscribe(user);
user.addSubject(subject);
}
@ -45,31 +82,43 @@ public class DataStore implements AutoCloseable {
}
}
}
private static void loadSubjectsFromFile() {
try (ObjectInputStream in = new ObjectInputStream(new FileInputStream(DATA_FILE))) {
subjectSubscriptions = (Map<Subject, Subscription>) in.readObject();
for (Map.Entry<Subject,Subscription> entry : subjectSubscriptions.entrySet()) {
System.out.println("Subject " + entry.getKey() + " has notified " + entry.getValue());
}
System.out.println("Loaded subjects from " + DATA_FILE);
try{
Thread.sleep(1000);
}catch (InterruptedException e){}
} catch (IOException | ClassNotFoundException e) {
System.out.println("Error loading subjects data: " + e.getMessage());
}
}
private static void saveSubjectsToFile() {
try (ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(DATA_FILE))) {
out.writeObject(subjectSubscriptions);
} catch (IOException e) {
System.out.println("Error saving subjects data: " + e.getMessage());
if (running) {
try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(DATA_FILE))) {
oos.writeObject(subjectSubscriptions);
System.out.println("DataStore state saved to file.");
} catch (IOException e) {
System.err.println("Failed to save DataStore state: " + e.getMessage());
}
}
}
@Override
public void close() throws Exception {
saveSubjectsToFile();
@SuppressWarnings("unchecked")
private static void loadSubjectsFromFile() {
File file = new File(DATA_FILE);
if(!file.exists()){
try {
file.createNewFile();
System.out.println("Created new file: " + DATA_FILE);
return;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
if (file.exists()) {
try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(file))) {
Object readObject = ois.readObject();
if (readObject instanceof Map) {
subjectSubscriptions = new ConcurrentHashMap<>((Map<Subject, Subscription>) readObject);
System.out.println("DataStore state loaded from file.");
}
} catch (IOException | ClassNotFoundException e) {
System.err.println("Failed to load DataStore state: " + e.getMessage());
}
}
}
}

View File

@ -5,15 +5,20 @@ import java.util.List;
import java.util.Set;
import java.io.*;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import com.fasterxml.jackson.databind.ObjectMapper;
public class DataThread implements Runnable {
private List<SubjectEntry> allEntries = new ArrayList<>();
private boolean running;
private Set<Subject> subjects = Subject.generateSubjects();
private static final String FILE_NAME = "allEntries.dat";
private static final String FILE_NAME = "allEntries.json";
private static final long SAVE_INTERVAL_MS = 10000;
private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
private static final ObjectMapper objectMapper = new ObjectMapper();
@Override
public void run() {
running = true;
@ -23,8 +28,8 @@ public class DataThread implements Runnable {
}catch (Exception e) {
e.printStackTrace();
}
while (running) {
executorService.scheduleAtFixedRate(this::saveEntriesToFile, 0, SAVE_INTERVAL_MS, TimeUnit.MILLISECONDS);
while (running && !Thread.currentThread().isInterrupted()) {
try {
System.out.println("comparing: " + compare(ETFScraper.getEntries()));
@ -32,27 +37,33 @@ public class DataThread implements Runnable {
} catch (InterruptedException e) {
System.out.println("Thread interrupted, stopping...");
running = false;
executorService.shutdown();
Thread.currentThread().interrupt();
return;
} catch (Exception e) {
e.printStackTrace();
}
}
saveEntriesToFile();
executorService.shutdown();
}
public boolean compare(List<SubjectEntry> subjectEntries) {
synchronized private boolean compare(List<SubjectEntry> subjectEntries) {
if (subjectEntries.size() == 0) {
System.out.println("No entries found");
return false;
}
for (SubjectEntry subjectEntry : subjectEntries) {
// If entry is new and not already present
if (!allEntries.contains(subjectEntry)) {
allEntries.add(subjectEntry);
for (Subject subject : subjects) {
if (subject.matchesTitle(subjectEntry.getTitle())) {
// addNewEntry(subject,subjectEntry);
DataStore.notifySubject(subject, subjectEntry);
allEntries.add(subjectEntry);
break;
}
}
@ -60,21 +71,27 @@ public class DataThread implements Runnable {
}
return true;
}
/*
synchronized private void addNewEntry(Subject subject, SubjectEntry subjectEntry) {
allEntries.add(subjectEntry);
DataStore.notifySubject(subject, subjectEntry);
}
*/
public void stop() {
running = false;
}
/*
private void saveEntriesToFile() {
try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(FILE_NAME))) {
oos.writeObject(allEntries);
System.out.println("Entries saved to file.");
} catch (IOException e) {
e.printStackTrace();
}
System.err.println("Failed to save entries: " + e.getMessage()); }
}
private void readEntriesFromFile() throws IOException {
File file = new File(FILE_NAME);
@ -108,4 +125,45 @@ public class DataThread implements Runnable {
e.printStackTrace();
}
}
*/
private void saveEntriesToFile() {
try {
// Serialize the entries to JSON
String json = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(allEntries);
// Write the JSON to a file
try (BufferedWriter writer = new BufferedWriter(new FileWriter(FILE_NAME))) {
writer.write(json);
System.out.println("All entries saved to JSON file.");
}
} catch (IOException e) {
System.err.println("Failed to save entries to JSON: " + e.getMessage());
}
}
private void readEntriesFromFile() {
File file = new File(FILE_NAME);
if (file.exists()) {
try (BufferedReader reader = new BufferedReader(new FileReader(file))) {
allEntries = objectMapper.readValue(reader, objectMapper.getTypeFactory().constructCollectionType(List.class, SubjectEntry.class));
System.out.println("Entries loaded from JSON file.");
} catch (IOException e) {
System.err.println("Failed to load entries from JSON: " + e.getMessage());
e.printStackTrace();
}
} else {
try {
// Create the file and write an empty list to it
file.createNewFile();
try (BufferedWriter writer = new BufferedWriter(new FileWriter(file))) {
writer.write("[]");
System.out.println("No previous entries found. Created an empty JSON file.");
}
} catch (IOException e) {
System.err.println("Failed to create empty JSON file: " + e.getMessage());
}
allEntries = new ArrayList<>();
}
}
}

View File

@ -1,7 +1,9 @@
package dev.ksan;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.IncorrectnessListener;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebClientOptions;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
@ -9,13 +11,23 @@ import com.gargoylesoftware.htmlunit.html.HtmlPage;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.ConsoleHandler;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
public class ETFScraper implements Runnable {
private static List<SubjectEntry> entries = new ArrayList<>();
private WebClient webClient;
private volatile boolean running = true;
public ETFScraper() {
this.webClient = new WebClient(BrowserVersion.CHROME);
webClient.getOptions().setJavaScriptEnabled(true);
webClient.getOptions().setCssEnabled(false);
webClient.getOptions().setThrowExceptionOnScriptError(false);
}
private static String getTextOrEmpty(HtmlElement parent, String xPath) {
HtmlElement element = parent.getFirstByXPath(xPath);
@ -26,15 +38,25 @@ public class ETFScraper implements Runnable {
return new ArrayList<>(entries);
}
}
private void configureHtmlUnitLogging() {
// Get the HtmlUnit logger and set its level to SEVERE to suppress warnings
Logger htmlUnitLogger = Logger.getLogger("com.gargoylesoftware.htmlunit");
htmlUnitLogger.setLevel(Level.SEVERE);
Handler consoleHandler = new ConsoleHandler();
consoleHandler.setLevel(Level.SEVERE);
htmlUnitLogger.addHandler(consoleHandler);
}
@Override
public void run() {
while(running) {
try (final WebClient webClient = new WebClient(BrowserVersion.CHROME)) {
configureHtmlUnitLogging();
while(running && !Thread.currentThread().isInterrupted()) {
try {
System.out.println("Performing WebClient task...");
webClient.getOptions().setJavaScriptEnabled(true);
webClient.getOptions().setCssEnabled(false);
webClient.getOptions().setThrowExceptionOnScriptError(false);
HtmlPage mainPage = webClient.getPage("https://efee.etf.unibl.org/oglasi/");
webClient.waitForBackgroundJavaScript(1000);
@ -74,7 +96,7 @@ public class ETFScraper implements Runnable {
for (HtmlElement pTag : pTags) {
paragraphs.add(pTag.asNormalizedText());
}
SubjectEntry entry = new SubjectEntry(title, date, info, paragraphs);
SubjectEntry entry = new SubjectEntry(title,groupName, date, info, paragraphs);
entries.add(entry);
@ -86,9 +108,12 @@ public class ETFScraper implements Runnable {
//Thread.sleep(20000);
} catch (Exception e) {
}
catch (Exception e) {
e.printStackTrace();
System.out.println("ERROR: " + e.getMessage());
} finally {
this.webClient.close();
}
}
System.out.println("WebScraper thread stopped");

View File

@ -1,25 +1,22 @@
package dev.ksan;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.*;
import java.io.IOException;
import java.util.*;
public class Main {
public static void main(String[] args) {
boolean running = true;
User user = new User("djordje@ksan.dev","123");
User user = Users.createUser("djordje@ksan.dev","123");
boolean istrue = user.addSubject("Filozofija");
user.addSubject("Programiranje 2");
System.out.println(istrue);
System.out.println(user.getEmail() + user.getId());
Set<Subject> subjectSet = user.getSubjectSet();
System.out.println(Users.getUserCount()+1);
List<SubjectEntry> entries = new ArrayList<>();
List<SubjectEntry> newEntries = new ArrayList<>();
@ -36,6 +33,7 @@ public class Main {
Scanner scanner = new Scanner(System.in);
dataThread.start();
DataStore.subscribeUserToSubject(user,new Subject("Osnovi elektrotehnike 2"));
try {
while (running) {
@ -48,6 +46,8 @@ public class Main {
task.stop();
webClientThread.interrupt();
running = false;
DataStore.stop();
Users.stop();
System.out.println("Stopping...");
break;
case "list":
@ -58,37 +58,13 @@ public class Main {
}
}catch (Exception e) {
scanner.close();
e.printStackTrace();
}
scanner.close();
//temp
SubjectEntry mail = new SubjectEntry("TAAAA","asd", "Test", Arrays.asList("Testing mail broj 2", "ne znam zas je u spamu"));
try {
//user.sendEmail(mail);
System.out.println("AAAAAAAAAAAAAAAAAAAA");
} catch (Exception e) {
throw new RuntimeException(e);
}finally {
scanner.close();
}
entries=task.getEntries();
System.out.println("BBBBBBBBBBBBb");
System.out.println(entries.size());
for(SubjectEntry e : task.getEntries()) {
//System.out.println(e);
}
for(Subject subject : user.getSubjectSet()) {
for(SubjectEntry entry : entries){
if(subject.isSubject(entry.getTitle())){
//user.sendEmail(entry);
System.out.println("Subject " + subject.getTitle() + " was found");
}
}
}
//temp
}
}

View File

@ -1,19 +1,32 @@
package dev.ksan;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.Serializable;
import java.util.List;
public class SubjectEntry {
public class SubjectEntry implements Serializable {
private String group;
private String title;
private String date;
private String info;
private List<String> paragraphs;
public SubjectEntry(String title, String date, String info, List<String> paragraphs) {
@JsonCreator
public SubjectEntry(@JsonProperty("title") String title,
@JsonProperty("group") String group,
@JsonProperty("date") String date,
@JsonProperty("info") String info,
@JsonProperty("paragraphs") List<String> paragraphs
) {
this.title = title;
this.group = group;
this.date = date;
this.info = info;
this.paragraphs = paragraphs;
}
public String getTitle() {
return title;
}
@ -23,6 +36,9 @@ public class SubjectEntry {
public String getInfo() {
return info;
}
public String getGroup() {
return group;
}
public List<String> getParagraphs() {
return paragraphs;
}
@ -38,7 +54,7 @@ public class SubjectEntry {
}
@Override
public String toString() {
return title + " " + date + " " + info + "\n\t" + paragraphs + "\n";
return title + " " + group + " " + date + " " + info + "\n\t" + paragraphs + "\n";
}
}

View File

@ -1,9 +1,10 @@
package dev.ksan;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class Subscription {
public class Subscription implements Serializable {
private Subject subject;
private List<User> users = new ArrayList<>();

View File

@ -8,9 +8,11 @@ import org.simplejavamail.email.EmailBuilder;
import org.simplejavamail.mailer.MailerBuilder;
import java.io.*;
import java.util.*;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
public class User implements Serializable {
private static Set<String> usedIds = new HashSet<>();
private String id;
private String email;
@ -18,8 +20,10 @@ public class User implements Serializable {
private NotificationMethod notificationMethod;
private Set<Subject> subjectSet = new HashSet<>();
public User(String email, String password) {
this.id = generateId();
public User(String email, String password, String id) {
this.id = id;
this.email = email;
this.password = password;
this.notificationMethod = NotificationMethod.EMAIL;
@ -28,39 +32,19 @@ public class User implements Serializable {
public void setNotificationMethod(NotificationMethod notificationMethod) {
this.notificationMethod = notificationMethod;
}
public void serializeToFile(String filename) throws IOException {
try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(filename))) {
oos.writeObject(this);
}
}
public static User deserializeFromFile(String filename) throws IOException, ClassNotFoundException {
try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(filename))) {
return (User) ois.readObject();
}
}
public static void saveUsedIds(String filename) throws IOException {
try (ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(filename))) {
oos.writeObject(usedIds);
}
}
@SuppressWarnings("unchecked")
public static void loadUsedIds(String filename) throws IOException, ClassNotFoundException {
try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(filename))) {
usedIds = (Set<String>) ois.readObject();
}
}
public void sendNotification(SubjectEntry entry){
if(notificationMethod == NotificationMethod.EMAIL){
System.out.println("Sending an e-mail notification");
// sendEmail(entry);
sendEmail(entry);
return;
} else if (notificationMethod == NotificationMethod.PUSH_NOTIFICATION) {
pushNotification(entry);
return;
}
System.out.println("Sending a notification failed");
}
private void pushNotification(SubjectEntry entry){
//TODO
@ -76,11 +60,11 @@ public class User implements Serializable {
}
public void addSubject(Subject subject) {
subjectSet.add(subject);
}
public void addSubjects(Set<Subject> subjects) {
this.subjectSet.addAll(subjects);
if(!subjectSet.contains(subject)){
subjectSet.add(subject);
}
}
public boolean addSubject(String subject) {
Set<Subject> subjects = new HashSet<>();
subjects = Subject.generateSubjects();
@ -107,13 +91,6 @@ public class User implements Serializable {
public void setPassword(String password) {
this.password = password;
}
private String generateId() {
String id;
do{
id = UUID.randomUUID().toString();
}while(usedIds.contains(id));
usedIds.add(id);
return id;
}
}

View File

@ -0,0 +1,114 @@
package dev.ksan;
import java.io.*;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
public class Users {
private static final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
private static final long SAVE_INTERVAL_MS = 10000;
private static final String USERS_FILE_NAME = "users.ser";
private static Map<String,User> users = new ConcurrentHashMap<>();
static{
loadUsersFromFile();
startPeriodicSave();
Runtime.getRuntime().addShutdownHook(new Thread(Users::stopPeriodicSave));
}
public static void stop(){
executorService.shutdownNow();
try {
if (!executorService.awaitTermination(3000, TimeUnit.MILLISECONDS)) {
executorService.shutdownNow();
}
}catch(InterruptedException e){
executorService.shutdownNow();
throw new RuntimeException("Error during manual shutdown");
}finally {
saveUsersToFile();
}
}
public static int getUserCount() {
return users.size();
}
public static User createUser(String email, String password) {
User user = new User(email, password, generateId());
addUser(user);
return user;
}
public static void addUser(User user){
users.put(user.getId(),user);
}
public static User getUser(String id){
return users.get(id);
}
public static void removeUser(String id){
users.remove(id);
}
private static void startPeriodicSave() {
executorService.scheduleAtFixedRate(Users::saveUsersToFile,0,SAVE_INTERVAL_MS, TimeUnit.MILLISECONDS);
}
public static void stopPeriodicSave() {
saveUsersToFile();
executorService.shutdown();
}
private static void saveUsersToFile() {
try(ObjectOutputStream oos = new ObjectOutputStream(new FileOutputStream(USERS_FILE_NAME))){
oos.writeObject(users);
System.out.println("Users saved to " + USERS_FILE_NAME);
}catch (IOException e){
e.printStackTrace();
System.err.println("Failed to save users to " + USERS_FILE_NAME);
}
}
private static void loadUsersFromFile() {
File file = new File(USERS_FILE_NAME);
if(!file.exists()){
try {
file.createNewFile();
System.out.println("Users file created: " + USERS_FILE_NAME);
} catch (IOException e) {
System.err.println("Failed to create users file" + USERS_FILE_NAME);
throw new RuntimeException(e);
}
return;
}
if(file.exists()){
try(ObjectInputStream ois = new ObjectInputStream(new FileInputStream(file))){
Object readObject = ois.readObject();
if(ois instanceof Map){
users.clear();
users.putAll((Map<String,User>) readObject);
System.out.println("Users loaded from " + USERS_FILE_NAME);
}
}catch(EOFException e){
System.err.println("Users file is empty");
users.clear();
System.err.println("Resetting users due to EOFException");
} catch (IOException | ClassNotFoundException e) {
System.err.println("Failed to load users from " + USERS_FILE_NAME);
e.printStackTrace();
throw new RuntimeException(e);
}
}
}
public static String generateId() {
String id;
do{
id = UUID.randomUUID().toString();
}while(users.containsKey(id));
return id;
}
}