refactoring to have full http response in parser
This commit is contained in:
parent
945a13ed2f
commit
0b9da1860f
33
owl_test.go
33
owl_test.go
|
@ -2,33 +2,44 @@ package owl_test
|
|||
|
||||
import (
|
||||
"h4kor/owl-blogs"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MockHttpParser struct{}
|
||||
type MockHtmlParser struct{}
|
||||
|
||||
func (*MockHttpParser) ParseHEntry(data []byte) (owl.ParsedHEntry, error) {
|
||||
func (*MockHtmlParser) ParseHEntry(resp *http.Response) (owl.ParsedHEntry, error) {
|
||||
return owl.ParsedHEntry{Title: "Mock Title"}, nil
|
||||
}
|
||||
|
||||
func (*MockHttpParser) ParseLinks(data []byte) ([]string, error) {
|
||||
}
|
||||
func (*MockHtmlParser) ParseLinks(resp *http.Response) ([]string, error) {
|
||||
return []string{"http://example.com"}, nil
|
||||
}
|
||||
|
||||
func (*MockHttpParser) GetWebmentionEndpoint(data []byte) (string, error) {
|
||||
}
|
||||
func (*MockHtmlParser) ParseLinksFromString(string) ([]string, error) {
|
||||
return []string{"http://example.com"}, nil
|
||||
|
||||
}
|
||||
func (*MockHtmlParser) GetWebmentionEndpoint(resp *http.Response) (string, error) {
|
||||
return "http://example.com/webmention", nil
|
||||
|
||||
}
|
||||
|
||||
type MockHttpRetriever struct{}
|
||||
type MockHttpClient struct{}
|
||||
|
||||
func (*MockHttpRetriever) Get(url string) ([]byte, error) {
|
||||
return []byte(""), nil
|
||||
func (*MockHttpClient) Get(url string) (resp *http.Response, err error) {
|
||||
return &http.Response{}, nil
|
||||
}
|
||||
func (*MockHttpClient) Post(url, contentType string, body io.Reader) (resp *http.Response, err error) {
|
||||
|
||||
func (m *MockHttpRetriever) Post(url string, data url.Values) ([]byte, error) {
|
||||
return []byte(""), nil
|
||||
return &http.Response{}, nil
|
||||
}
|
||||
func (*MockHttpClient) PostForm(url string, data url.Values) (resp *http.Response, err error) {
|
||||
|
||||
return &http.Response{}, nil
|
||||
}
|
||||
|
||||
func randomName() string {
|
||||
|
|
12
post.go
12
post.go
|
@ -295,13 +295,13 @@ func (post *Post) UpdateOutgoingWebmention(webmention *WebmentionOut) error {
|
|||
}
|
||||
|
||||
func (post *Post) EnrichWebmention(source string) error {
|
||||
html, err := post.user.repo.HttpClient.Get(source)
|
||||
resp, err := post.user.repo.HttpClient.Get(source)
|
||||
if err == nil {
|
||||
webmention, err := post.Webmention(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry, err := post.user.repo.Parser.ParseHEntry(html)
|
||||
entry, err := post.user.repo.Parser.ParseHEntry(resp)
|
||||
if err == nil {
|
||||
webmention.Title = entry.Title
|
||||
return post.PersistWebmention(webmention)
|
||||
|
@ -359,7 +359,7 @@ func (post *Post) ScanForLinks() error {
|
|||
// this could be done in markdown parsing, but I don't want to
|
||||
// rely on goldmark for this (yet)
|
||||
postHtml := post.RenderedContent()
|
||||
links, _ := post.user.repo.Parser.ParseLinks(postHtml.Bytes())
|
||||
links, _ := post.user.repo.Parser.ParseLinksFromString(string(postHtml.Bytes()))
|
||||
for _, link := range links {
|
||||
post.AddOutgoingWebmention(link)
|
||||
}
|
||||
|
@ -370,13 +370,13 @@ func (post *Post) SendWebmention(webmention WebmentionOut) error {
|
|||
defer post.UpdateOutgoingWebmention(&webmention)
|
||||
webmention.ScannedAt = time.Now()
|
||||
|
||||
html, err := post.user.repo.HttpClient.Get(webmention.Target)
|
||||
resp, err := post.user.repo.HttpClient.Get(webmention.Target)
|
||||
if err != nil {
|
||||
webmention.Supported = false
|
||||
return err
|
||||
}
|
||||
|
||||
endpoint, err := post.user.repo.Parser.GetWebmentionEndpoint(html)
|
||||
endpoint, err := post.user.repo.Parser.GetWebmentionEndpoint(resp)
|
||||
if err != nil {
|
||||
webmention.Supported = false
|
||||
return err
|
||||
|
@ -387,7 +387,7 @@ func (post *Post) SendWebmention(webmention WebmentionOut) error {
|
|||
payload := url.Values{}
|
||||
payload.Set("source", post.FullUrl())
|
||||
payload.Set("target", webmention.Target)
|
||||
_, err = post.user.repo.HttpClient.Post(endpoint, payload)
|
||||
_, err = post.user.repo.HttpClient.PostForm(endpoint, payload)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
16
post_test.go
16
post_test.go
|
@ -190,8 +190,8 @@ func TestPersistWebmention(t *testing.T) {
|
|||
|
||||
func TestAddWebmentionCreatesFile(t *testing.T) {
|
||||
repo := getTestRepo(owl.RepoConfig{})
|
||||
repo.HttpClient = &MockHttpRetriever{}
|
||||
repo.Parser = &MockHttpParser{}
|
||||
repo.HttpClient = &MockHttpClient{}
|
||||
repo.Parser = &MockHtmlParser{}
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
|
@ -208,8 +208,8 @@ func TestAddWebmentionCreatesFile(t *testing.T) {
|
|||
|
||||
func TestAddWebmentionNotOverwritingFile(t *testing.T) {
|
||||
repo := getTestRepo(owl.RepoConfig{})
|
||||
repo.HttpClient = &MockHttpRetriever{}
|
||||
repo.Parser = &MockHttpParser{}
|
||||
repo.HttpClient = &MockHttpClient{}
|
||||
repo.Parser = &MockHtmlParser{}
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
|
@ -238,8 +238,8 @@ func TestAddWebmentionNotOverwritingFile(t *testing.T) {
|
|||
|
||||
func TestAddWebmentionAddsParsedTitle(t *testing.T) {
|
||||
repo := getTestRepo(owl.RepoConfig{})
|
||||
repo.HttpClient = &MockHttpRetriever{}
|
||||
repo.Parser = &MockHttpParser{}
|
||||
repo.HttpClient = &MockHttpClient{}
|
||||
repo.Parser = &MockHtmlParser{}
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
|
@ -353,8 +353,8 @@ func TestScanningForLinksDoesNotAddDuplicates(t *testing.T) {
|
|||
|
||||
func TestCanSendWebmention(t *testing.T) {
|
||||
repo := getTestRepo(owl.RepoConfig{})
|
||||
repo.HttpClient = &MockHttpRetriever{}
|
||||
repo.Parser = &MockHttpParser{}
|
||||
repo.HttpClient = &MockHttpClient{}
|
||||
repo.Parser = &MockHtmlParser{}
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
|
|
|
@ -28,7 +28,7 @@ type RepoConfig struct {
|
|||
}
|
||||
|
||||
func CreateRepository(name string, config RepoConfig) (Repository, error) {
|
||||
newRepo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}}
|
||||
newRepo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: &OwlHttpClient{}}
|
||||
// check if repository already exists
|
||||
if dirExists(newRepo.Dir()) {
|
||||
return Repository{}, fmt.Errorf("Repository already exists")
|
||||
|
@ -69,7 +69,7 @@ func CreateRepository(name string, config RepoConfig) (Repository, error) {
|
|||
|
||||
func OpenRepository(name string) (Repository, error) {
|
||||
|
||||
repo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}}
|
||||
repo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: &OwlHttpClient{}}
|
||||
if !dirExists(repo.Dir()) {
|
||||
return Repository{}, fmt.Errorf("Repository does not exist: " + repo.Dir())
|
||||
}
|
||||
|
|
|
@ -3,7 +3,6 @@ package owl
|
|||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
@ -28,17 +27,19 @@ type WebmentionOut struct {
|
|||
}
|
||||
|
||||
type HttpClient interface {
|
||||
Get(url string) ([]byte, error)
|
||||
Post(url string, data url.Values) ([]byte, error)
|
||||
Get(url string) (resp *http.Response, err error)
|
||||
Post(url, contentType string, body io.Reader) (resp *http.Response, err error)
|
||||
PostForm(url string, data url.Values) (resp *http.Response, err error)
|
||||
}
|
||||
|
||||
type HtmlParser interface {
|
||||
ParseHEntry(data []byte) (ParsedHEntry, error)
|
||||
ParseLinks(data []byte) ([]string, error)
|
||||
GetWebmentionEndpoint(data []byte) (string, error)
|
||||
ParseHEntry(resp *http.Response) (ParsedHEntry, error)
|
||||
ParseLinks(resp *http.Response) ([]string, error)
|
||||
ParseLinksFromString(string) ([]string, error)
|
||||
GetWebmentionEndpoint(resp *http.Response) (string, error)
|
||||
}
|
||||
|
||||
type OwlHttpClient struct{}
|
||||
type OwlHttpClient = http.Client
|
||||
|
||||
type OwlHtmlParser struct{}
|
||||
|
||||
|
@ -46,30 +47,8 @@ type ParsedHEntry struct {
|
|||
Title string
|
||||
}
|
||||
|
||||
func (OwlHttpClient) Get(url string) ([]byte, error) {
|
||||
resp, err := http.Get(url)
|
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return make([]byte, 0), errors.New("Failed to get url. Status code: " + fmt.Sprint(resp.StatusCode))
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
func (OwlHttpClient) Post(url string, data url.Values) ([]byte, error) {
|
||||
resp, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
func collectText(n *html.Node, buf *bytes.Buffer) {
|
||||
|
||||
if n.Type == html.TextNode {
|
||||
buf.WriteString(n.Data)
|
||||
}
|
||||
|
@ -78,8 +57,18 @@ func collectText(n *html.Node, buf *bytes.Buffer) {
|
|||
}
|
||||
}
|
||||
|
||||
func (OwlHtmlParser) ParseHEntry(data []byte) (ParsedHEntry, error) {
|
||||
doc, err := html.Parse(strings.NewReader(string(data)))
|
||||
func readResponseBody(resp *http.Response) (string, error) {
|
||||
defer resp.Body.Close()
|
||||
bodyBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(bodyBytes), nil
|
||||
}
|
||||
|
||||
func (OwlHtmlParser) ParseHEntry(resp *http.Response) (ParsedHEntry, error) {
|
||||
htmlStr, err := readResponseBody(resp)
|
||||
doc, err := html.Parse(strings.NewReader(htmlStr))
|
||||
if err != nil {
|
||||
return ParsedHEntry{}, err
|
||||
}
|
||||
|
@ -121,8 +110,16 @@ func (OwlHtmlParser) ParseHEntry(data []byte) (ParsedHEntry, error) {
|
|||
return findHFeed(doc)
|
||||
}
|
||||
|
||||
func (OwlHtmlParser) ParseLinks(data []byte) ([]string, error) {
|
||||
doc, err := html.Parse(strings.NewReader(string(data)))
|
||||
func (OwlHtmlParser) ParseLinks(resp *http.Response) ([]string, error) {
|
||||
htmlStr, err := readResponseBody(resp)
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
}
|
||||
return OwlHtmlParser{}.ParseLinksFromString(htmlStr)
|
||||
}
|
||||
|
||||
func (OwlHtmlParser) ParseLinksFromString(htmlStr string) ([]string, error) {
|
||||
doc, err := html.Parse(strings.NewReader(htmlStr))
|
||||
if err != nil {
|
||||
return make([]string, 0), err
|
||||
}
|
||||
|
@ -144,11 +141,11 @@ func (OwlHtmlParser) ParseLinks(data []byte) ([]string, error) {
|
|||
return links, nil
|
||||
}
|
||||
return findLinks(doc)
|
||||
|
||||
}
|
||||
|
||||
func (OwlHtmlParser) GetWebmentionEndpoint(data []byte) (string, error) {
|
||||
doc, err := html.Parse(strings.NewReader(string(data)))
|
||||
func (OwlHtmlParser) GetWebmentionEndpoint(resp *http.Response) (string, error) {
|
||||
htmlStr, err := readResponseBody(resp)
|
||||
doc, err := html.Parse(strings.NewReader(htmlStr))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
@ -1,7 +1,10 @@
|
|||
package owl_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"h4kor/owl-blogs"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
@ -12,7 +15,7 @@ import (
|
|||
func TestParseValidHEntry(t *testing.T) {
|
||||
html := []byte("<div class=\"h-entry\"><div class=\"p-name\">Foo</div></div>")
|
||||
parser := &owl.OwlHtmlParser{}
|
||||
entry, err := parser.ParseHEntry(html)
|
||||
entry, err := parser.ParseHEntry(&http.Response{Body: io.NopCloser(bytes.NewReader(html))})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Unable to parse feed: %v", err)
|
||||
|
@ -25,7 +28,7 @@ func TestParseValidHEntry(t *testing.T) {
|
|||
func TestParseValidHEntryWithoutTitle(t *testing.T) {
|
||||
html := []byte("<div class=\"h-entry\"></div><div class=\"p-name\">Foo</div>")
|
||||
parser := &owl.OwlHtmlParser{}
|
||||
entry, err := parser.ParseHEntry(html)
|
||||
entry, err := parser.ParseHEntry(&http.Response{Body: io.NopCloser(bytes.NewReader(html))})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Unable to parse feed: %v", err)
|
||||
|
@ -38,7 +41,7 @@ func TestParseValidHEntryWithoutTitle(t *testing.T) {
|
|||
func TestGetWebmentionEndpointLink(t *testing.T) {
|
||||
html := []byte("<link rel=\"webmention\" href=\"http://example.com/webmention\" />")
|
||||
parser := &owl.OwlHtmlParser{}
|
||||
endpoint, err := parser.GetWebmentionEndpoint(html)
|
||||
endpoint, err := parser.GetWebmentionEndpoint(&http.Response{Body: io.NopCloser(bytes.NewReader(html))})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Unable to parse feed: %v", err)
|
||||
|
@ -51,7 +54,7 @@ func TestGetWebmentionEndpointLink(t *testing.T) {
|
|||
func TestGetWebmentionEndpointLinkA(t *testing.T) {
|
||||
html := []byte("<a rel=\"webmention\" href=\"http://example.com/webmention\" />")
|
||||
parser := &owl.OwlHtmlParser{}
|
||||
endpoint, err := parser.GetWebmentionEndpoint(html)
|
||||
endpoint, err := parser.GetWebmentionEndpoint(&http.Response{Body: io.NopCloser(bytes.NewReader(html))})
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("Unable to parse feed: %v", err)
|
||||
|
@ -60,3 +63,43 @@ func TestGetWebmentionEndpointLinkA(t *testing.T) {
|
|||
t.Errorf("Wrong endpoint. Expected %v, got %v", "http://example.com/webmention", endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
// func TestRealWorldWebmention(t *testing.T) {
|
||||
// links := []string{
|
||||
// "https://webmention.rocks/test/1",
|
||||
// "https://webmention.rocks/test/2",
|
||||
// "https://webmention.rocks/test/3",
|
||||
// "https://webmention.rocks/test/4",
|
||||
// "https://webmention.rocks/test/5",
|
||||
// "https://webmention.rocks/test/6",
|
||||
// "https://webmention.rocks/test/7",
|
||||
// "https://webmention.rocks/test/8",
|
||||
// "https://webmention.rocks/test/9",
|
||||
// "https://webmention.rocks/test/10",
|
||||
// "https://webmention.rocks/test/11",
|
||||
// "https://webmention.rocks/test/12",
|
||||
// "https://webmention.rocks/test/13",
|
||||
// "https://webmention.rocks/test/14",
|
||||
// "https://webmention.rocks/test/15",
|
||||
// "https://webmention.rocks/test/16",
|
||||
// "https://webmention.rocks/test/17",
|
||||
// "https://webmention.rocks/test/18",
|
||||
// "https://webmention.rocks/test/19",
|
||||
// "https://webmention.rocks/test/20",
|
||||
// "https://webmention.rocks/test/21",
|
||||
// "https://webmention.rocks/test/22",
|
||||
// "https://webmention.rocks/test/23/page",
|
||||
// }
|
||||
|
||||
// for _, link := range links {
|
||||
// parser := &owl.OwlHtmlParser{}
|
||||
// client := &owl.OwlHttpClient{}
|
||||
// html, _ := client.Get(link)
|
||||
// _, err := parser.GetWebmentionEndpoint(html)
|
||||
|
||||
// if err != nil {
|
||||
// t.Errorf("Unable to find webmention: %v for link %v", err, link)
|
||||
// }
|
||||
// }
|
||||
|
||||
// }
|
||||
|
|
Loading…
Reference in New Issue