From dadbfde8024644b3ec83a0cfabf873ce142271d6 Mon Sep 17 00:00:00 2001 From: Ksan Date: Sat, 9 May 2026 03:15:41 +0200 Subject: [PATCH] fixed --- init.sql | 1 + .../config/SecurityConfig.java | 3 +- .../controller/EntryController.java | 47 +++++++++----- .../controller/UserController.java | 18 ++--- .../ksan/etfoglasiserver/dto/EntryDTO.java | 18 +++-- .../dev/ksan/etfoglasiserver/model/Entry.java | 56 +++++++++------- .../ksan/etfoglasiserver/model/Subject.java | 3 + .../etfoglasiserver/repository/EntryRepo.java | 18 ++++- .../etfoglasiserver/service/EntryService.java | 65 +++++++++++++++++-- .../ksan/etfoglasiserver/service/Scraper.java | 40 +++++++++--- .../etfoglasiserver/util/HashGenerator.java | 2 +- src/main/resources/application.properties | 1 + 12 files changed, 204 insertions(+), 68 deletions(-) diff --git a/init.sql b/init.sql index 1427af9..dda85d4 100644 --- a/init.sql +++ b/init.sql @@ -21,6 +21,7 @@ CREATE TABLE IF NOT EXISTS public.entries paragraph text COLLATE pg_catalog."default", time_published timestamp without time zone NOT NULL, filepath text COLLATE pg_catalog."default", + group_name text COLLATE pg_catalog."default", CONSTRAINT entries_pkey PRIMARY KEY (id) ); diff --git a/src/main/java/dev/ksan/etfoglasiserver/config/SecurityConfig.java b/src/main/java/dev/ksan/etfoglasiserver/config/SecurityConfig.java index 4aa5174..c468b7f 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/config/SecurityConfig.java +++ b/src/main/java/dev/ksan/etfoglasiserver/config/SecurityConfig.java @@ -31,7 +31,8 @@ public class SecurityConfig { return http.csrf(customizer -> customizer.disable()). authorizeHttpRequests(request -> request - .requestMatchers("login", "register").permitAll() + .requestMatchers("/login", "/register").permitAll() + .requestMatchers("/subject/**", "/entries", "/groups").permitAll() .anyRequest().authenticated()). httpBasic(Customizer.withDefaults()). sessionManagement(session -> session.sessionCreationPolicy(SessionCreationPolicy.STATELESS)) diff --git a/src/main/java/dev/ksan/etfoglasiserver/controller/EntryController.java b/src/main/java/dev/ksan/etfoglasiserver/controller/EntryController.java index 02f4da0..3ed50ef 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/controller/EntryController.java +++ b/src/main/java/dev/ksan/etfoglasiserver/controller/EntryController.java @@ -6,39 +6,56 @@ import dev.ksan.etfoglasiserver.service.EntryService; import java.util.List; import java.util.UUID; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.DeleteMapping; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.data.web.PageableDefault; +import org.springframework.web.bind.annotation.*; @RestController public class EntryController { @Autowired EntryService service; - @GetMapping("/entry") - public List getEntries() { - return service.getEntries(); - } +// @GetMapping("/entries") +// public List getEntries() { +// return service.getEntries(); +// } - @GetMapping("/entry/{entryId}") + @GetMapping("/entries/{entryId}") public Entry getEntry(@PathVariable String entryId) { return service.getEntryById(entryId); } - @PostMapping("/entry") + //ignore this for now + @PostMapping("/entries") public void addEntry(@RequestBody EntryDTO entry) { service.addEntry(entry); } - @PutMapping("/entry") + + @GetMapping("/groups") + public List getGroups() { + return service.getAllGroups(); + } + + + @GetMapping("/entries") + public Page getEntries( + @RequestParam(required = false) String subjectId, + @RequestParam(required = false) String groupName, + @RequestParam(defaultValue = "0") int page + ) { + UUID parsedSubjectId = subjectId != null ? UUID.fromString(subjectId) : null; + return service.getEntries(parsedSubjectId, groupName, page); + } + + @PutMapping("/entries") public void updateEntry(@RequestBody EntryDTO entry) { service.updateEntry(entry); } - @DeleteMapping("/entry/{entryId}") + @DeleteMapping("/entries/{entryId}") public void deleteEntry(@PathVariable String entryId) { service.deleteEntry(entryId); } diff --git a/src/main/java/dev/ksan/etfoglasiserver/controller/UserController.java b/src/main/java/dev/ksan/etfoglasiserver/controller/UserController.java index 6e0db07..af0aebb 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/controller/UserController.java +++ b/src/main/java/dev/ksan/etfoglasiserver/controller/UserController.java @@ -47,14 +47,7 @@ public class UserController { return new ResponseEntity<>(HttpStatus.BAD_REQUEST); } - /* - @PostMapping("/users") - public void addUser(@RequestBody UserCreationDTO user) { - NotificationMethod method = NotificationMethod.valueOf(user.getNotification()); - user.setNotificationMethod(method); - service.addUser(user); - } - */ + @PostMapping("/login") public ResponseEntity login(@RequestBody UserLoginDTO user) { JwtResponseDTO userVer = service.verify(user); @@ -75,4 +68,13 @@ public class UserController { public void deleteUser(@PathVariable UUID userId) { service.deleteUser(userId); } + + /* + @PostMapping("/users") + public void addUser(@RequestBody UserCreationDTO user) { + NotificationMethod method = NotificationMethod.valueOf(user.getNotification()); + user.setNotificationMethod(method); + service.addUser(user); + } + */ } diff --git a/src/main/java/dev/ksan/etfoglasiserver/dto/EntryDTO.java b/src/main/java/dev/ksan/etfoglasiserver/dto/EntryDTO.java index 1bbeecb..fbdc060 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/dto/EntryDTO.java +++ b/src/main/java/dev/ksan/etfoglasiserver/dto/EntryDTO.java @@ -6,8 +6,9 @@ import java.util.UUID; public class EntryDTO { - private String id; + private UUID id; private String title; + private String group; private LocalDateTime time_published; private String info_entry; private String paragraph; @@ -15,8 +16,9 @@ public class EntryDTO { private UUID subjectId; public EntryDTO() {} - public EntryDTO( String title, LocalDateTime time_published, String info_entry, String paragraph, String filepath, UUID subjectId) { + public EntryDTO( String title, String group, LocalDateTime time_published, String info_entry, String paragraph, String filepath, UUID subjectId) { this.title = title; + this.group = group; this.time_published = time_published; this.info_entry = info_entry; this.paragraph = paragraph; @@ -38,11 +40,11 @@ public class EntryDTO { return subjectId; } - public String getId() { + public UUID getId() { return id; } - public void setId(String id) { + public void setId(UUID id) { this.id = id; } public void setSubjectId(UUID subjectId) { @@ -92,4 +94,12 @@ public class EntryDTO { public void setFilepath(String filepath) { this.filepath = filepath; } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } } diff --git a/src/main/java/dev/ksan/etfoglasiserver/model/Entry.java b/src/main/java/dev/ksan/etfoglasiserver/model/Entry.java index c5d26c4..f6971a2 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/model/Entry.java +++ b/src/main/java/dev/ksan/etfoglasiserver/model/Entry.java @@ -3,6 +3,7 @@ package dev.ksan.etfoglasiserver.model; import dev.ksan.etfoglasiserver.dto.EntryDTO; import dev.ksan.etfoglasiserver.util.HashGenerator; import jakarta.persistence.*; + import java.time.LocalDateTime; import java.util.List; import java.util.UUID; @@ -10,14 +11,23 @@ import java.util.UUID; @Entity @Table(name = "entries") public class Entry { - @Id private String id; + @Id + @GeneratedValue + @Column(columnDefinition = "uuid", updatable = false, nullable = false) + private UUID id; + @ManyToOne(fetch = FetchType.EAGER) @JoinColumn(nullable = false) private Subject subject; @Column private String title; - @Column private LocalDateTime time_published; + @Column(name = "time_published") + private LocalDateTime timePublished; + + @Column(name = "group_name") + private String groupName; + private String info_entry; private String paragraph; private String filepath; @@ -31,7 +41,7 @@ public class Entry { String paragraph, String filepath) { this.title = title; - this.time_published = time_published; + this.timePublished= time_published; this.info_entry = info_entry; this.paragraph = paragraph; this.filepath = filepath; @@ -44,14 +54,15 @@ public class Entry { List paragraph, String filepath) { this.title = title; - this.time_published = time_published; + this.timePublished= time_published; this.info_entry = info_entry; this.paragraph = String.join("\n", paragraph); this.filepath = filepath; } - public Entry( String title, LocalDateTime time_published, String info_entry, List paragraphs, String filepath, Subject subjectId) { + public Entry( String title,String group, LocalDateTime time_published, String info_entry, List paragraphs, String filepath, Subject subjectId) { this.title = title; - this.time_published = time_published; + this.groupName = group; + this.timePublished= time_published; this.info_entry = info_entry; this.paragraph = String.join("\n",paragraphs); this.filepath = filepath; @@ -63,16 +74,7 @@ public class Entry { this.paragraph = entry.getParagraph(); this.info_entry = entry.getInfo_entry(); this.filepath = entry.getFilepath(); - this.time_published = entry.getTime_published(); - } - - @PrePersist - public void generateId() { - - if (this.id == null) { - - this.id = HashGenerator.hashEntry(this); - } + this.timePublished= entry.getTime_published(); } public String getParagraph() { @@ -103,21 +105,23 @@ public class Entry { return title; } - public LocalDateTime getDate() { - return time_published; + public LocalDateTime getTimePublished() { + return timePublished; } public String getInfo() { return info_entry; } - + public UUID getId() { + return id; + } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null) return false; Entry subject = (Entry) o; if (title.equals(subject.getTitle()) - && time_published.equals(subject.getDate()) + && timePublished.equals(subject.getTimePublished()) && info_entry.equals(subject.getInfo())) { return true; } @@ -135,7 +139,7 @@ public class Entry { + title + '\'' + ", time_published=" - + time_published + + timePublished + ", info_entry='" + info_entry + '\'' @@ -161,6 +165,14 @@ public class Entry { } public void setTime_published(LocalDateTime timePublished) { - this.time_published = timePublished; + this.timePublished= timePublished; } + + public String getGroup() { + return groupName; + } + + public void setGroup(String group) { + this.groupName = group; + } } diff --git a/src/main/java/dev/ksan/etfoglasiserver/model/Subject.java b/src/main/java/dev/ksan/etfoglasiserver/model/Subject.java index 9aabec4..7b7a658 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/model/Subject.java +++ b/src/main/java/dev/ksan/etfoglasiserver/model/Subject.java @@ -11,6 +11,7 @@ public class Subject { @Id @GeneratedValue(strategy = GenerationType.AUTO) + @Column(columnDefinition = "uuid") private UUID id; @Column private long code; @@ -18,6 +19,8 @@ public class Subject { @Column(nullable = false) private String name; + + @ManyToMany(mappedBy = "subjectSet", fetch = FetchType.LAZY) Set users; public Subject(){ diff --git a/src/main/java/dev/ksan/etfoglasiserver/repository/EntryRepo.java b/src/main/java/dev/ksan/etfoglasiserver/repository/EntryRepo.java index bdbc46d..1a6b393 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/repository/EntryRepo.java +++ b/src/main/java/dev/ksan/etfoglasiserver/repository/EntryRepo.java @@ -1,9 +1,25 @@ package dev.ksan.etfoglasiserver.repository; import dev.ksan.etfoglasiserver.model.Entry; + +import java.time.LocalDateTime; +import java.util.Optional; import java.util.UUID; + +import dev.ksan.etfoglasiserver.model.Subject; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Repository; @Repository -public interface EntryRepo extends JpaRepository {} +public interface EntryRepo extends JpaRepository { + + Optional findByTitleAndTimePublishedAndSubject(String title, LocalDateTime timePublished, Subject subject); + + Page findAllByOrderByTimePublishedDesc(Pageable pageable); + Page findBySubjectOrderByTimePublishedDesc(Subject subject, Pageable pageable); + Page findByGroupNameOrderByTimePublishedDesc(String groupName, Pageable pageable); + Page findBySubjectAndGroupNameOrderByTimePublishedDesc(Subject subject, String groupName, Pageable pageable); +} + diff --git a/src/main/java/dev/ksan/etfoglasiserver/service/EntryService.java b/src/main/java/dev/ksan/etfoglasiserver/service/EntryService.java index 5aa0852..570d4cb 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/service/EntryService.java +++ b/src/main/java/dev/ksan/etfoglasiserver/service/EntryService.java @@ -6,16 +6,26 @@ import dev.ksan.etfoglasiserver.model.Subject; import dev.ksan.etfoglasiserver.repository.EntryRepo; import dev.ksan.etfoglasiserver.repository.SubjectRepo; import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.UUID; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Sort; +import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; @Service public class EntryService { - @Autowired EntryRepo entryRepo; + @Autowired + EntryRepo entryRepo; - @Autowired SubjectService subjectService; + @Autowired + SubjectService subjectService; public List getEntries() { return entryRepo.findAll(); @@ -28,18 +38,28 @@ public class EntryService { public void addEntry(EntryDTO entry) { Subject subject = - subjectService - .findById(entry.getSubjectId()); + subjectService + .findById(entry.getSubjectId()); Entry newEntry = new Entry(entry); newEntry.setSubject(subject); entryRepo.save(newEntry); } - public void addEntry(Entry entry) { + public void addEntry(Entry entry) { + Optional existing = entryRepo.findByTitleAndTimePublishedAndSubject( + entry.getTitle(), + entry.getTimePublished(), + entry.getSubject() + ); + + if (existing.isPresent()) { + System.out.println("Duplicate entry, skipping"); + return; + } try { entryRepo.save(entry); - }catch (Exception e) { + } catch (Exception e) { System.out.println("Duplicate"); return; } @@ -50,7 +70,7 @@ public class EntryService { Subject subject = subjectService .findById(entry.getSubjectId()); - Entry updateEntry = entryRepo.findById(entry.getId()).orElseThrow(() -> new RuntimeException("Entry not found")); + Entry updateEntry = entryRepo.findById(entry.getId().toString()).orElseThrow(() -> new RuntimeException("Entry not found")); updateEntry.setSubject(subject); if (entry.getTitle() != null) { updateEntry.setTitle(entry.getTitle()); @@ -73,4 +93,35 @@ public class EntryService { public void deleteEntry(String id) { entryRepo.deleteById(id); } + + + public Page getEntries(UUID subjectId, String groupName, int page) { + Pageable pageable = PageRequest.of(page, 20, Sort.by("timePublished").descending()); + + if (subjectId != null && groupName != null) { + Subject subject = subjectService.findById(subjectId); + return entryRepo.findBySubjectAndGroupNameOrderByTimePublishedDesc(subject, groupName, pageable); + } + if (subjectId != null) { + Subject subject = subjectService.findById(subjectId); + return entryRepo.findBySubjectOrderByTimePublishedDesc(subject, pageable); + } + if (groupName != null) { + return entryRepo.findByGroupNameOrderByTimePublishedDesc(groupName, pageable); + } + + return entryRepo.findAllByOrderByTimePublishedDesc(pageable); + } + + public List getAllGroups() { + List allEntries = entryRepo.findAll(); + + // Extract unique group names + return allEntries.stream() + .map(Entry::getGroup) + .filter(Objects::nonNull) + .distinct() + .sorted() + .toList(); + } } diff --git a/src/main/java/dev/ksan/etfoglasiserver/service/Scraper.java b/src/main/java/dev/ksan/etfoglasiserver/service/Scraper.java index ba68b3d..1cde60d 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/service/Scraper.java +++ b/src/main/java/dev/ksan/etfoglasiserver/service/Scraper.java @@ -2,10 +2,7 @@ package dev.ksan.etfoglasiserver.service; import com.gargoylesoftware.htmlunit.BrowserVersion; import com.gargoylesoftware.htmlunit.WebClient; -import com.gargoylesoftware.htmlunit.html.DomElement; -import com.gargoylesoftware.htmlunit.html.HtmlAnchor; -import com.gargoylesoftware.htmlunit.html.HtmlElement; -import com.gargoylesoftware.htmlunit.html.HtmlPage; +import com.gargoylesoftware.htmlunit.html.*; import dev.ksan.etfoglasiserver.dto.EntryDTO; import dev.ksan.etfoglasiserver.model.Entry; import java.time.LocalDateTime; @@ -74,8 +71,23 @@ public class Scraper implements Runnable { } int ul_idSelection = 1; for (HtmlAnchor anchor : toggles) { - String groupName = anchor.asNormalizedText().split("\n")[0].trim(); - // System.out.println("Group name: " + groupName); + HtmlElement span = anchor.getFirstByXPath(".//span[@class='ui-btn-text']"); + if (span == null) continue; // skip anchors with no text span + + // Skip “clear text” buttons + String spanText = span.asNormalizedText().toLowerCase(); + if (spanText.contains("clear text")) continue; + + // Only get direct text nodes (skip child span with collapse text) + List textNodes = span.getByXPath("./text()"); + StringBuilder sb = new StringBuilder(); + for (DomText textNode : textNodes) { + sb.append(textNode.asNormalizedText()); + } + String groupName = sb.toString().trim(); + + System.out.println("Group name: " + groupName); + HtmlPage updatedPage = anchor.click(); webClient.waitForBackgroundJavaScript(1000); @@ -98,15 +110,25 @@ public class Scraper implements Runnable { String info = getTextOrEmpty(item, ".//h2[2]"); List paragraphs = new ArrayList<>(); List pTags = item.getByXPath(".//p"); + String filepath = null; // default for (HtmlElement pTag : pTags) { paragraphs.add(pTag.asNormalizedText()); + + // check if this

contains an link + HtmlAnchor aTag = pTag.getFirstByXPath(".//a"); + if (aTag != null) { + filepath = aTag.getHrefAttribute(); + } } - // Entry entry = new Entry(title, groupName, date, info, paragraphs); + Subject subjectid = altService.findSubjectIdWithTitle(title); Entry entry = new Entry( - title.trim(), LocalDateTime.parse(date, formatter), info, paragraphs, null, subjectid); + title.trim(), groupName, LocalDateTime.parse(date, formatter), + info, paragraphs, filepath, subjectid); + System.out.println("Subject "+entry.getTitle()); + System.out.println(); entryService.addEntry(entry); @@ -115,7 +137,7 @@ public class Scraper implements Runnable { ul_idSelection++; } - // Thread.sleep(20000); + Thread.sleep(300000); } catch (Exception e) { e.printStackTrace(); diff --git a/src/main/java/dev/ksan/etfoglasiserver/util/HashGenerator.java b/src/main/java/dev/ksan/etfoglasiserver/util/HashGenerator.java index 0d98311..dcf2a8d 100644 --- a/src/main/java/dev/ksan/etfoglasiserver/util/HashGenerator.java +++ b/src/main/java/dev/ksan/etfoglasiserver/util/HashGenerator.java @@ -10,7 +10,7 @@ public class HashGenerator { public static String hashEntry(Entry entry) { String data = Objects.toString(entry.getTitle(), "") - + Objects.toString(entry.getDate(), "") + + Objects.toString(entry.getTimePublished(), "") + Objects.toString(entry.getParagraph(), "") + Objects.toString(entry.getFilepath(), ""); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index af20303..ca491f4 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -2,3 +2,4 @@ spring.application.name=etfoglasi-server spring.datasource.url=jdbc:postgresql://localhost:5432/etfo-db spring.datasource.username=test spring.datasource.password=test +