evan/AnthoLume
evan/AnthoLume
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: annas archive parsing
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Evan Reichard 2025-08-17 17:04:03 -04:00
parent 7937890acd
commit 3cff965393
1 changed files with 10 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
search/anna.go
View File

@ -4,14 +4,11 @@ import (
"fmt" "fmt"
"io" "io"
"net/url" "net/url"
"regexp"
"strings" "strings"
"github.com/PuerkitoBio/goquery" "github.com/PuerkitoBio/goquery"
) )
var commentRE = regexp.MustCompile(`(?s)<!--(.*?)-->`)
func searchAnnasArchive(query string) ([]SearchItem, error) { func searchAnnasArchive(query string) ([]SearchItem, error) {
searchURL := "https://annas-archive.org/search?index=&q=%s&ext=epub&sort=&lang=en" searchURL := "https://annas-archive.org/search?index=&q=%s&ext=epub&sort=&lang=en"
url := fmt.Sprintf(searchURL, url.QueryEscape(query)) url := fmt.Sprintf(searchURL, url.QueryEscape(query))
@ -32,62 +29,34 @@ func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
// Normalize Results // Normalize Results
var allEntries []SearchItem var allEntries []SearchItem
doc.Find("#aarecord-list > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) { doc.Find(".js-aarecord-list-outer > div > div").Each(func(ix int, rawBook *goquery.Selection) {
rawBook = getAnnasArchiveBookSelection(rawBook)
// Parse Details // Parse Details
details := rawBook.Find("div:nth-child(2) > div:nth-child(1)").Text() details := rawBook.Find("div:nth-child(3)").Text()
detailsSplit := strings.Split(details, ", ") detailsSplit := strings.Split(details, " · ")
// Invalid Details // Invalid Details
if len(detailsSplit) < 4 { if len(detailsSplit) < 3 {
return return
} }
// Parse MD5 // Parse MD5
itemHref, _ := rawBook.Find("a").Attr("href") titleAuthorDetails := rawBook.Find("div a")
titleEl := titleAuthorDetails.Eq(0)
itemHref, _ := titleEl.Attr("href")
hrefArray := strings.Split(itemHref, "/") hrefArray := strings.Split(itemHref, "/")
id := hrefArray[len(hrefArray)-1] id := hrefArray[len(hrefArray)-1]
allEntries = append(allEntries, SearchItem{ allEntries = append(allEntries, SearchItem{
ID: id, ID: id,
Title: rawBook.Find("h3").First().Text(), Title: titleEl.Text(),
Author: rawBook.Find("div:nth-child(2) > div:nth-child(4)").First().Text(), Author: titleAuthorDetails.Eq(1).Text(),
Language: detailsSplit[0], Language: detailsSplit[0],
FileType: detailsSplit[1], FileType: detailsSplit[1],
FileSize: detailsSplit[3], FileSize: detailsSplit[2],
}) })
}) })
// Return Results // Return Results
return allEntries, nil return allEntries, nil
} }
// getAnnasArchiveBookSelection parses potentially commented out HTML. For some reason
// Annas Archive comments out blocks "below the fold". They aren't rendered until you
// scroll. This attempts to parse the commented out HTML.
func getAnnasArchiveBookSelection(rawBook *goquery.Selection) *goquery.Selection {
rawHTML, err := rawBook.Html()
if err != nil {
return rawBook
}
strippedHTML := strings.TrimSpace(rawHTML)
if !strings.HasPrefix(strippedHTML, "<!--") || !strings.HasSuffix(strippedHTML, "-->") {
return rawBook
}
allMatches := commentRE.FindAllStringSubmatch(strippedHTML, -1)
if len(allMatches) != 1 || len(allMatches[0]) != 2 {
return rawBook
}
captureGroup := allMatches[0][1]
docReader := strings.NewReader(captureGroup)
doc, err := goquery.NewDocumentFromReader(docReader)
if err != nil {
return rawBook
}
return doc.Selection
}