fix: search parsing

search/anna.go

@@ -32,7 +32,7 @@ func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
 
 	// Normalize Results
 	var allEntries []SearchItem
-	doc.Find("form > div.w-full > div.w-full > div > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) {
+	doc.Find("#aarecord-list > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) {
 		rawBook = getAnnasArchiveBookSelection(rawBook)
 
 		// Parse Details
@@ -44,29 +44,19 @@ func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
 			return
 		}
 
-		language := detailsSplit[0]
-		fileType := detailsSplit[1]
-		fileSize := detailsSplit[3]
-
-		// Get Title & Author
-		title := rawBook.Find("h3").Text()
-		author := rawBook.Find("div:nth-child(2) > div:nth-child(4)").Text()
-
 		// Parse MD5
 		itemHref, _ := rawBook.Find("a").Attr("href")
 		hrefArray := strings.Split(itemHref, "/")
 		id := hrefArray[len(hrefArray)-1]
 
-		item := SearchItem{
+		allEntries = append(allEntries, SearchItem{
 			ID:       id,
-			Title:    title,
+			Title:    rawBook.Find("h3").First().Text(),
-			Author:   author,
+			Author:   rawBook.Find("div:nth-child(2) > div:nth-child(4)").First().Text(),
-			Language: language,
+			Language: detailsSplit[0],
-			FileType: fileType,
+			FileType: detailsSplit[1],
-			FileSize: fileSize,
+			FileSize: detailsSplit[3],
-		}
+		})
-
-		allEntries = append(allEntries, item)
 	})
 
 	// Return Results

search/downloaders.go

@@ -25,7 +25,7 @@ func getLibGenDownloadURL(md5 string, _ Source) ([]string, error) {
 	// Return Download URL
 	downloadPath, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href")
 	if !exists {
-		return nil, fmt.Errorf("Download URL not found")
+		return nil, fmt.Errorf("download URL not found")
 	}
 
 	// Possible Funky URL
@@ -37,10 +37,10 @@ func getLibraryDownloadURL(md5 string, source Source) ([]string, error) {
 	// Derive Info URL
 	var infoURL string
 	switch source {
-	case SOURCE_LIBGEN_FICTION, SOURCE_ANNAS_ARCHIVE:
+	case SOURCE_LIBGEN, SOURCE_ANNAS_ARCHIVE:
 		infoURL = "http://library.lol/fiction/" + md5
-	case SOURCE_LIBGEN_NON_FICTION:
+	// case SOURCE_LIBGEN_NON_FICTION:
-		infoURL = "http://library.lol/main/" + md5
+	// 	infoURL = "http://library.lol/main/" + md5
 	default:
 		return nil, errors.New("invalid source")
 	}
@@ -62,7 +62,7 @@ func getLibraryDownloadURL(md5 string, source Source) ([]string, error) {
 	// downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href")
 	downloadURL, exists := doc.Find("#download h2 a").Attr("href")
 	if !exists {
-		return nil, errors.New("Download URL not found")
+		return nil, errors.New("download URL not found")
 	}
 
 	return []string{downloadURL}, nil

search/libgen.go

@@ -1,89 +1,44 @@
 package search
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"net/url"
 	"strings"
-	"time"
 
 	"github.com/PuerkitoBio/goquery"
 )
 
-func searchLibGenFiction(query string) ([]SearchItem, error) {
+const LIBGEN_SEARCH_URL = "https://%s/index.php?req=ext:epub+%s&gmode=on"
-	searchURL := "https://libgen.is/fiction/?q=%s&language=English&format=epub"
+
-	url := fmt.Sprintf(searchURL, url.QueryEscape(query))
+var libgenDomains []string = []string{
-	body, err := getPage(url)
+	"libgen.vg",
-	if err != nil {
+	"libgen.is",
-		return nil, err
-	}
-	return parseLibGenFiction(body)
 }
 
-func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) {
+func searchLibGen(query string) ([]SearchItem, error) {
-	// Parse
+	var allErrors []error
-	defer body.Close()
-	doc, err := goquery.NewDocumentFromReader(body)
-	if err != nil {
-		return nil, err
-	}
 
-	// Normalize Results
+	for _, domain := range libgenDomains {
-	var allEntries []SearchItem
+		url := fmt.Sprintf(LIBGEN_SEARCH_URL, domain, url.QueryEscape(query))
-	doc.Find("table.catalog tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
+		body, err := getPage(url)
-
+		if err != nil {
-		// Parse File Details
+			allErrors = append(allErrors, err)
-		fileItem := rawBook.Find("td:nth-child(5)")
+			continue
-		fileDesc := fileItem.Text()
-		fileDescSplit := strings.Split(fileDesc, "/")
-		fileType := strings.ToLower(strings.TrimSpace(fileDescSplit[0]))
-		fileSize := strings.TrimSpace(fileDescSplit[1])
-
-		// Parse Upload Date
-		uploadedRaw, _ := fileItem.Attr("title")
-		uploadedDateRaw := strings.Split(uploadedRaw, "Uploaded at ")[1]
-		uploadDate, _ := time.Parse("2006-01-02 15:04:05", uploadedDateRaw)
-
-		// Parse MD5
-		editHref, _ := rawBook.Find("td:nth-child(7) a").Attr("href")
-		hrefArray := strings.Split(editHref, "/")
-		id := hrefArray[len(hrefArray)-1]
-
-		// Parse Other Details
-		title := rawBook.Find("td:nth-child(3) p a").Text()
-		author := rawBook.Find(".catalog_authors li a").Text()
-		language := rawBook.Find("td:nth-child(4)").Text()
-		series := rawBook.Find("td:nth-child(2)").Text()
-
-		item := SearchItem{
-			ID:         id,
-			Title:      title,
-			Author:     author,
-			Series:     series,
-			Language:   language,
-			FileType:   fileType,
-			FileSize:   fileSize,
-			UploadDate: uploadDate.Format(time.RFC3339),
 		}
-
+		results, err := parseLibGen(body)
-		allEntries = append(allEntries, item)
+		if err != nil {
-	})
+			allErrors = append(allErrors, err)
-
+			continue
-	// Return Results
+		}
-	return allEntries, nil
+		return results, nil
-}
-
-func searchLibGenNonFiction(query string) ([]SearchItem, error) {
-	searchURL := "https://libgen.is/search.php?req=%s"
-	url := fmt.Sprintf(searchURL, url.QueryEscape(query))
-	body, err := getPage(url)
-	if err != nil {
-		return nil, err
 	}
-	return parseLibGenNonFiction(body)
+
+	return nil, fmt.Errorf("could not query libgen: %w", errors.Join(allErrors...))
 }
 
-func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) {
+func parseLibGen(body io.ReadCloser) ([]SearchItem, error) {
 	// Parse
 	defer body.Close()
 	doc, err := goquery.NewDocumentFromReader(body)
@@ -93,35 +48,25 @@ func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) {
 
 	// Normalize Results
 	var allEntries []SearchItem
-	doc.Find("table.c tbody > tr:nth-child(n + 2)").Each(func(ix int, rawBook *goquery.Selection) {
+	doc.Find("#tablelibgen tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
-
-		// Parse Type & Size
-		fileSize := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text()))
-		fileType := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(9)").Text()))
-
 		// Parse MD5
-		titleRaw := rawBook.Find("td:nth-child(3) [id]")
+		linksRaw := rawBook.Find("td:nth-child(9) a")
-		editHref, _ := titleRaw.Attr("href")
+		linksHref, _ := linksRaw.Attr("href")
-		hrefArray := strings.Split(editHref, "?md5=")
+		hrefArray := strings.Split(linksHref, "?md5=")
+		if len(hrefArray) == 0 {
+			return
+		}
 		id := hrefArray[1]
 
-		// Parse Other Details
+		allEntries = append(allEntries, SearchItem{
-		title := titleRaw.Text()
-		author := rawBook.Find("td:nth-child(2)").Text()
-		language := rawBook.Find("td:nth-child(7)").Text()
-		series := rawBook.Find("td:nth-child(3) [href*='column=series']").Text()
-
-		item := SearchItem{
 			ID:       id,
-			Title:    title,
+			Title:    rawBook.Find("td:nth-child(1) > a").First().Text(),
-			Author:   author,
+			Author:   rawBook.Find("td:nth-child(2)").Text(),
-			Series:   series,
+			Series:   rawBook.Find("td:nth-child(1) > b").Text(),
-			Language: language,
+			Language: rawBook.Find("td:nth-child(5)").Text(),
-			FileType: fileType,
+			FileType: strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text())),
-			FileSize: fileSize,
+			FileSize: strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(7)").Text())),
-		}
+		})
-
-		allEntries = append(allEntries, item)
 	})
 
 	// Return Results

search/search.go

@@ -25,9 +25,8 @@ const (
 type Source string
 
 const (
-	SOURCE_ANNAS_ARCHIVE      Source = "Annas Archive"
+	SOURCE_ANNAS_ARCHIVE Source = "Annas Archive"
-	SOURCE_LIBGEN_FICTION     Source = "LibGen Fiction"
+	SOURCE_LIBGEN        Source = "LibGen"
-	SOURCE_LIBGEN_NON_FICTION Source = "LibGen Non-fiction"
 )
 
 type SearchItem struct {
@@ -45,9 +44,8 @@ type searchFunc func(query string) (searchResults []SearchItem, err error)
 type downloadFunc func(md5 string, source Source) (downloadURL []string, err error)
 
 var searchDefs = map[Source]searchFunc{
-	SOURCE_ANNAS_ARCHIVE:      searchAnnasArchive,
+	SOURCE_ANNAS_ARCHIVE: searchAnnasArchive,
-	SOURCE_LIBGEN_FICTION:     searchLibGenFiction,
+	SOURCE_LIBGEN:        searchLibGen,
-	SOURCE_LIBGEN_NON_FICTION: searchLibGenNonFiction,
 }
 
 var downloadFuncs = []downloadFunc{

templates/pages/search.tmpl

@@ -35,9 +35,8 @@
               id="source"
               name="source"
             >
+              <option value="LibGen">Library Genesis</option>
               <option value="Annas Archive">Annas Archive</option>
-              <option value="LibGen Fiction">LibGen Fiction</option>
-              <option value="LibGen Non-fiction">LibGen Non-fiction</option>
             </select>
           </div>
           <div class="lg:w-60">
@@ -133,7 +132,8 @@
                   </form>
                 </td>
                 <td class="p-3 border-b border-gray-200">
-                  {{ $item.Author }} - {{ $item.Title }}
+                  {{ $item.Author }} -
+                  {{ $item.Title }}
                 </td>
                 <td class="p-3 border-b border-gray-200">
                   <p>{{ or $item.Series "N/A" }}</p>