fix: search parsing

2025-07-05 16:46:06 -04:00 · 2025-07-05 16:46:06 -04:00 · 0fe52bc541
commit 0fe52bc541
parent 49f3d53170
5 changed files with 58 additions and 125 deletions
--- a/search/anna.go
+++ b/search/anna.go
@ -32,7 +32,7 @@ func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {

 	// Normalize Results
 	var allEntries []SearchItem
-	doc.Find("form > div.w-full > div.w-full > div > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) {
+	doc.Find("#aarecord-list > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) {
 		rawBook = getAnnasArchiveBookSelection(rawBook)

 		// Parse Details
@ -44,29 +44,19 @@ func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
 			return
 		}

-		language := detailsSplit[0]
-		fileType := detailsSplit[1]
-		fileSize := detailsSplit[3]
-
-		// Get Title & Author
-		title := rawBook.Find("h3").Text()
-		author := rawBook.Find("div:nth-child(2) > div:nth-child(4)").Text()
-
 		// Parse MD5
 		itemHref, _ := rawBook.Find("a").Attr("href")
 		hrefArray := strings.Split(itemHref, "/")
 		id := hrefArray[len(hrefArray)-1]

-		item := SearchItem{
+		allEntries = append(allEntries, SearchItem{
 			ID:       id,
-			Title:    title,
-			Author:   author,
-			Language: language,
-			FileType: fileType,
-			FileSize: fileSize,
-		}
-
-		allEntries = append(allEntries, item)
+			Title:    rawBook.Find("h3").First().Text(),
+			Author:   rawBook.Find("div:nth-child(2) > div:nth-child(4)").First().Text(),
+			Language: detailsSplit[0],
+			FileType: detailsSplit[1],
+			FileSize: detailsSplit[3],
+		})
 	})

 	// Return Results
--- a/search/downloaders.go
+++ b/search/downloaders.go
@ -25,7 +25,7 @@ func getLibGenDownloadURL(md5 string, _ Source) ([]string, error) {
 	// Return Download URL
 	downloadPath, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href")
 	if !exists {
-		return nil, fmt.Errorf("Download URL not found")
+		return nil, fmt.Errorf("download URL not found")
 	}

 	// Possible Funky URL
@ -37,10 +37,10 @@ func getLibraryDownloadURL(md5 string, source Source) ([]string, error) {
 	// Derive Info URL
 	var infoURL string
 	switch source {
-	case SOURCE_LIBGEN_FICTION, SOURCE_ANNAS_ARCHIVE:
+	case SOURCE_LIBGEN, SOURCE_ANNAS_ARCHIVE:
 		infoURL = "http://library.lol/fiction/" + md5
-	case SOURCE_LIBGEN_NON_FICTION:
-		infoURL = "http://library.lol/main/" + md5
+	// case SOURCE_LIBGEN_NON_FICTION:
+	// 	infoURL = "http://library.lol/main/" + md5
 	default:
 		return nil, errors.New("invalid source")
 	}
@ -62,7 +62,7 @@ func getLibraryDownloadURL(md5 string, source Source) ([]string, error) {
 	// downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href")
 	downloadURL, exists := doc.Find("#download h2 a").Attr("href")
 	if !exists {
-		return nil, errors.New("Download URL not found")
+		return nil, errors.New("download URL not found")
 	}

 	return []string{downloadURL}, nil
--- a/search/libgen.go
+++ b/search/libgen.go
@ -1,26 +1,44 @@
 package search

 import (
+	"errors"
 	"fmt"
 	"io"
 	"net/url"
 	"strings"
-	"time"

 	"github.com/PuerkitoBio/goquery"
 )

-func searchLibGenFiction(query string) ([]SearchItem, error) {
-	searchURL := "https://libgen.is/fiction/?q=%s&language=English&format=epub"
-	url := fmt.Sprintf(searchURL, url.QueryEscape(query))
-	body, err := getPage(url)
-	if err != nil {
-		return nil, err
-	}
-	return parseLibGenFiction(body)
+const LIBGEN_SEARCH_URL = "https://%s/index.php?req=ext:epub+%s&gmode=on"
+
+var libgenDomains []string = []string{
+	"libgen.vg",
+	"libgen.is",
 }

-func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) {
+func searchLibGen(query string) ([]SearchItem, error) {
+	var allErrors []error
+
+	for _, domain := range libgenDomains {
+		url := fmt.Sprintf(LIBGEN_SEARCH_URL, domain, url.QueryEscape(query))
+		body, err := getPage(url)
+		if err != nil {
+			allErrors = append(allErrors, err)
+			continue
+		}
+		results, err := parseLibGen(body)
+		if err != nil {
+			allErrors = append(allErrors, err)
+			continue
+		}
+		return results, nil
+	}
+
+	return nil, fmt.Errorf("could not query libgen: %w", errors.Join(allErrors...))
+}
+
+func parseLibGen(body io.ReadCloser) ([]SearchItem, error) {
 	// Parse
 	defer body.Close()
 	doc, err := goquery.NewDocumentFromReader(body)
@ -30,98 +48,25 @@ func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) {

 	// Normalize Results
 	var allEntries []SearchItem
-	doc.Find("table.catalog tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
-
-		// Parse File Details
-		fileItem := rawBook.Find("td:nth-child(5)")
-		fileDesc := fileItem.Text()
-		fileDescSplit := strings.Split(fileDesc, "/")
-		fileType := strings.ToLower(strings.TrimSpace(fileDescSplit[0]))
-		fileSize := strings.TrimSpace(fileDescSplit[1])
-
-		// Parse Upload Date
-		uploadedRaw, _ := fileItem.Attr("title")
-		uploadedDateRaw := strings.Split(uploadedRaw, "Uploaded at ")[1]
-		uploadDate, _ := time.Parse("2006-01-02 15:04:05", uploadedDateRaw)
-
+	doc.Find("#tablelibgen tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
 		// Parse MD5
-		editHref, _ := rawBook.Find("td:nth-child(7) a").Attr("href")
-		hrefArray := strings.Split(editHref, "/")
-		id := hrefArray[len(hrefArray)-1]
-
-		// Parse Other Details
-		title := rawBook.Find("td:nth-child(3) p a").Text()
-		author := rawBook.Find(".catalog_authors li a").Text()
-		language := rawBook.Find("td:nth-child(4)").Text()
-		series := rawBook.Find("td:nth-child(2)").Text()
-
-		item := SearchItem{
-			ID:         id,
-			Title:      title,
-			Author:     author,
-			Series:     series,
-			Language:   language,
-			FileType:   fileType,
-			FileSize:   fileSize,
-			UploadDate: uploadDate.Format(time.RFC3339),
+		linksRaw := rawBook.Find("td:nth-child(9) a")
+		linksHref, _ := linksRaw.Attr("href")
+		hrefArray := strings.Split(linksHref, "?md5=")
+		if len(hrefArray) == 0 {
+			return
 		}
-
-		allEntries = append(allEntries, item)
-	})
-
-	// Return Results
-	return allEntries, nil
-}
-
-func searchLibGenNonFiction(query string) ([]SearchItem, error) {
-	searchURL := "https://libgen.is/search.php?req=%s"
-	url := fmt.Sprintf(searchURL, url.QueryEscape(query))
-	body, err := getPage(url)
-	if err != nil {
-		return nil, err
-	}
-	return parseLibGenNonFiction(body)
-}
-
-func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) {
-	// Parse
-	defer body.Close()
-	doc, err := goquery.NewDocumentFromReader(body)
-	if err != nil {
-		return nil, err
-	}
-
-	// Normalize Results
-	var allEntries []SearchItem
-	doc.Find("table.c tbody > tr:nth-child(n + 2)").Each(func(ix int, rawBook *goquery.Selection) {
-
-		// Parse Type & Size
-		fileSize := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text()))
-		fileType := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(9)").Text()))
-
-		// Parse MD5
-		titleRaw := rawBook.Find("td:nth-child(3) [id]")
-		editHref, _ := titleRaw.Attr("href")
-		hrefArray := strings.Split(editHref, "?md5=")
 		id := hrefArray[1]

-		// Parse Other Details
-		title := titleRaw.Text()
-		author := rawBook.Find("td:nth-child(2)").Text()
-		language := rawBook.Find("td:nth-child(7)").Text()
-		series := rawBook.Find("td:nth-child(3) [href*='column=series']").Text()
-
-		item := SearchItem{
+		allEntries = append(allEntries, SearchItem{
 			ID:       id,
-			Title:    title,
-			Author:   author,
-			Series:   series,
-			Language: language,
-			FileType: fileType,
-			FileSize: fileSize,
-		}
-
-		allEntries = append(allEntries, item)
+			Title:    rawBook.Find("td:nth-child(1) > a").First().Text(),
+			Author:   rawBook.Find("td:nth-child(2)").Text(),
+			Series:   rawBook.Find("td:nth-child(1) > b").Text(),
+			Language: rawBook.Find("td:nth-child(5)").Text(),
+			FileType: strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text())),
+			FileSize: strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(7)").Text())),
+		})
 	})

 	// Return Results
--- a/search/search.go
+++ b/search/search.go
@ -26,8 +26,7 @@ type Source string

 const (
 	SOURCE_ANNAS_ARCHIVE Source = "Annas Archive"
-	SOURCE_LIBGEN_FICTION     Source = "LibGen Fiction"
-	SOURCE_LIBGEN_NON_FICTION Source = "LibGen Non-fiction"
+	SOURCE_LIBGEN        Source = "LibGen"
 )

 type SearchItem struct {
@ -46,8 +45,7 @@ type downloadFunc func(md5 string, source Source) (downloadURL []string, err err

 var searchDefs = map[Source]searchFunc{
 	SOURCE_ANNAS_ARCHIVE: searchAnnasArchive,
-	SOURCE_LIBGEN_FICTION:     searchLibGenFiction,
-	SOURCE_LIBGEN_NON_FICTION: searchLibGenNonFiction,
+	SOURCE_LIBGEN:        searchLibGen,
 }

 var downloadFuncs = []downloadFunc{
--- a/templates/pages/search.tmpl
+++ b/templates/pages/search.tmpl
@ -35,9 +35,8 @@
              id="source"
              name="source"
            >
+              <option value="LibGen">Library Genesis</option>
              <option value="Annas Archive">Annas Archive</option>
-              <option value="LibGen Fiction">LibGen Fiction</option>
-              <option value="LibGen Non-fiction">LibGen Non-fiction</option>
            </select>
          </div>
          <div class="lg:w-60">
@ -133,7 +132,8 @@
                  </form>
                </td>
                <td class="p-3 border-b border-gray-200">
-                  {{ $item.Author }} - {{ $item.Title }}
+                  {{ $item.Author }} -
+                  {{ $item.Title }}
                </td>
                <td class="p-3 border-b border-gray-200">
                  <p>{{ or $item.Series "N/A" }}</p>