diff --git a/owl_test.go b/owl_test.go index 532d21f..8fb1159 100644 --- a/owl_test.go +++ b/owl_test.go @@ -2,33 +2,44 @@ package owl_test import ( "h4kor/owl-blogs" + "io" "math/rand" + "net/http" "net/url" "time" ) -type MockHttpParser struct{} +type MockHtmlParser struct{} -func (*MockHttpParser) ParseHEntry(data []byte) (owl.ParsedHEntry, error) { +func (*MockHtmlParser) ParseHEntry(resp *http.Response) (owl.ParsedHEntry, error) { return owl.ParsedHEntry{Title: "Mock Title"}, nil -} -func (*MockHttpParser) ParseLinks(data []byte) ([]string, error) { +} +func (*MockHtmlParser) ParseLinks(resp *http.Response) ([]string, error) { return []string{"http://example.com"}, nil -} -func (*MockHttpParser) GetWebmentionEndpoint(data []byte) (string, error) { +} +func (*MockHtmlParser) ParseLinksFromString(string) ([]string, error) { + return []string{"http://example.com"}, nil + +} +func (*MockHtmlParser) GetWebmentionEndpoint(resp *http.Response) (string, error) { return "http://example.com/webmention", nil + } -type MockHttpRetriever struct{} +type MockHttpClient struct{} -func (*MockHttpRetriever) Get(url string) ([]byte, error) { - return []byte(""), nil +func (*MockHttpClient) Get(url string) (resp *http.Response, err error) { + return &http.Response{}, nil } +func (*MockHttpClient) Post(url, contentType string, body io.Reader) (resp *http.Response, err error) { -func (m *MockHttpRetriever) Post(url string, data url.Values) ([]byte, error) { - return []byte(""), nil + return &http.Response{}, nil +} +func (*MockHttpClient) PostForm(url string, data url.Values) (resp *http.Response, err error) { + + return &http.Response{}, nil } func randomName() string { diff --git a/post.go b/post.go index 1259f5a..2136697 100644 --- a/post.go +++ b/post.go @@ -295,13 +295,13 @@ func (post *Post) UpdateOutgoingWebmention(webmention *WebmentionOut) error { } func (post *Post) EnrichWebmention(source string) error { - html, err := post.user.repo.HttpClient.Get(source) + resp, err := post.user.repo.HttpClient.Get(source) if err == nil { webmention, err := post.Webmention(source) if err != nil { return err } - entry, err := post.user.repo.Parser.ParseHEntry(html) + entry, err := post.user.repo.Parser.ParseHEntry(resp) if err == nil { webmention.Title = entry.Title return post.PersistWebmention(webmention) @@ -359,7 +359,7 @@ func (post *Post) ScanForLinks() error { // this could be done in markdown parsing, but I don't want to // rely on goldmark for this (yet) postHtml := post.RenderedContent() - links, _ := post.user.repo.Parser.ParseLinks(postHtml.Bytes()) + links, _ := post.user.repo.Parser.ParseLinksFromString(string(postHtml.Bytes())) for _, link := range links { post.AddOutgoingWebmention(link) } @@ -370,13 +370,13 @@ func (post *Post) SendWebmention(webmention WebmentionOut) error { defer post.UpdateOutgoingWebmention(&webmention) webmention.ScannedAt = time.Now() - html, err := post.user.repo.HttpClient.Get(webmention.Target) + resp, err := post.user.repo.HttpClient.Get(webmention.Target) if err != nil { webmention.Supported = false return err } - endpoint, err := post.user.repo.Parser.GetWebmentionEndpoint(html) + endpoint, err := post.user.repo.Parser.GetWebmentionEndpoint(resp) if err != nil { webmention.Supported = false return err @@ -387,7 +387,7 @@ func (post *Post) SendWebmention(webmention WebmentionOut) error { payload := url.Values{} payload.Set("source", post.FullUrl()) payload.Set("target", webmention.Target) - _, err = post.user.repo.HttpClient.Post(endpoint, payload) + _, err = post.user.repo.HttpClient.PostForm(endpoint, payload) if err != nil { return err diff --git a/post_test.go b/post_test.go index 2825d3c..d26272c 100644 --- a/post_test.go +++ b/post_test.go @@ -190,8 +190,8 @@ func TestPersistWebmention(t *testing.T) { func TestAddWebmentionCreatesFile(t *testing.T) { repo := getTestRepo(owl.RepoConfig{}) - repo.HttpClient = &MockHttpRetriever{} - repo.Parser = &MockHttpParser{} + repo.HttpClient = &MockHttpClient{} + repo.Parser = &MockHtmlParser{} user, _ := repo.CreateUser("testuser") post, _ := user.CreateNewPost("testpost") @@ -208,8 +208,8 @@ func TestAddWebmentionCreatesFile(t *testing.T) { func TestAddWebmentionNotOverwritingFile(t *testing.T) { repo := getTestRepo(owl.RepoConfig{}) - repo.HttpClient = &MockHttpRetriever{} - repo.Parser = &MockHttpParser{} + repo.HttpClient = &MockHttpClient{} + repo.Parser = &MockHtmlParser{} user, _ := repo.CreateUser("testuser") post, _ := user.CreateNewPost("testpost") @@ -238,8 +238,8 @@ func TestAddWebmentionNotOverwritingFile(t *testing.T) { func TestAddWebmentionAddsParsedTitle(t *testing.T) { repo := getTestRepo(owl.RepoConfig{}) - repo.HttpClient = &MockHttpRetriever{} - repo.Parser = &MockHttpParser{} + repo.HttpClient = &MockHttpClient{} + repo.Parser = &MockHtmlParser{} user, _ := repo.CreateUser("testuser") post, _ := user.CreateNewPost("testpost") @@ -353,8 +353,8 @@ func TestScanningForLinksDoesNotAddDuplicates(t *testing.T) { func TestCanSendWebmention(t *testing.T) { repo := getTestRepo(owl.RepoConfig{}) - repo.HttpClient = &MockHttpRetriever{} - repo.Parser = &MockHttpParser{} + repo.HttpClient = &MockHttpClient{} + repo.Parser = &MockHtmlParser{} user, _ := repo.CreateUser("testuser") post, _ := user.CreateNewPost("testpost") diff --git a/repository.go b/repository.go index cf97038..4c35124 100644 --- a/repository.go +++ b/repository.go @@ -28,7 +28,7 @@ type RepoConfig struct { } func CreateRepository(name string, config RepoConfig) (Repository, error) { - newRepo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}} + newRepo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: &OwlHttpClient{}} // check if repository already exists if dirExists(newRepo.Dir()) { return Repository{}, fmt.Errorf("Repository already exists") @@ -69,7 +69,7 @@ func CreateRepository(name string, config RepoConfig) (Repository, error) { func OpenRepository(name string) (Repository, error) { - repo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}} + repo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: &OwlHttpClient{}} if !dirExists(repo.Dir()) { return Repository{}, fmt.Errorf("Repository does not exist: " + repo.Dir()) } diff --git a/webmention.go b/webmention.go index 8b4eb70..0e47c43 100644 --- a/webmention.go +++ b/webmention.go @@ -3,7 +3,6 @@ package owl import ( "bytes" "errors" - "fmt" "io" "net/http" "net/url" @@ -28,17 +27,19 @@ type WebmentionOut struct { } type HttpClient interface { - Get(url string) ([]byte, error) - Post(url string, data url.Values) ([]byte, error) + Get(url string) (resp *http.Response, err error) + Post(url, contentType string, body io.Reader) (resp *http.Response, err error) + PostForm(url string, data url.Values) (resp *http.Response, err error) } type HtmlParser interface { - ParseHEntry(data []byte) (ParsedHEntry, error) - ParseLinks(data []byte) ([]string, error) - GetWebmentionEndpoint(data []byte) (string, error) + ParseHEntry(resp *http.Response) (ParsedHEntry, error) + ParseLinks(resp *http.Response) ([]string, error) + ParseLinksFromString(string) ([]string, error) + GetWebmentionEndpoint(resp *http.Response) (string, error) } -type OwlHttpClient struct{} +type OwlHttpClient = http.Client type OwlHtmlParser struct{} @@ -46,30 +47,8 @@ type ParsedHEntry struct { Title string } -func (OwlHttpClient) Get(url string) ([]byte, error) { - resp, err := http.Get(url) - - if resp.StatusCode < 200 || resp.StatusCode > 299 { - return make([]byte, 0), errors.New("Failed to get url. Status code: " + fmt.Sprint(resp.StatusCode)) - } - - if err != nil { - return []byte{}, err - } - defer resp.Body.Close() - return io.ReadAll(resp.Body) -} - -func (OwlHttpClient) Post(url string, data url.Values) ([]byte, error) { - resp, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) - if err != nil { - return []byte{}, err - } - defer resp.Body.Close() - return io.ReadAll(resp.Body) -} - func collectText(n *html.Node, buf *bytes.Buffer) { + if n.Type == html.TextNode { buf.WriteString(n.Data) } @@ -78,8 +57,18 @@ func collectText(n *html.Node, buf *bytes.Buffer) { } } -func (OwlHtmlParser) ParseHEntry(data []byte) (ParsedHEntry, error) { - doc, err := html.Parse(strings.NewReader(string(data))) +func readResponseBody(resp *http.Response) (string, error) { + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(bodyBytes), nil +} + +func (OwlHtmlParser) ParseHEntry(resp *http.Response) (ParsedHEntry, error) { + htmlStr, err := readResponseBody(resp) + doc, err := html.Parse(strings.NewReader(htmlStr)) if err != nil { return ParsedHEntry{}, err } @@ -121,8 +110,16 @@ func (OwlHtmlParser) ParseHEntry(data []byte) (ParsedHEntry, error) { return findHFeed(doc) } -func (OwlHtmlParser) ParseLinks(data []byte) ([]string, error) { - doc, err := html.Parse(strings.NewReader(string(data))) +func (OwlHtmlParser) ParseLinks(resp *http.Response) ([]string, error) { + htmlStr, err := readResponseBody(resp) + if err != nil { + return []string{}, err + } + return OwlHtmlParser{}.ParseLinksFromString(htmlStr) +} + +func (OwlHtmlParser) ParseLinksFromString(htmlStr string) ([]string, error) { + doc, err := html.Parse(strings.NewReader(htmlStr)) if err != nil { return make([]string, 0), err } @@ -144,11 +141,11 @@ func (OwlHtmlParser) ParseLinks(data []byte) ([]string, error) { return links, nil } return findLinks(doc) - } -func (OwlHtmlParser) GetWebmentionEndpoint(data []byte) (string, error) { - doc, err := html.Parse(strings.NewReader(string(data))) +func (OwlHtmlParser) GetWebmentionEndpoint(resp *http.Response) (string, error) { + htmlStr, err := readResponseBody(resp) + doc, err := html.Parse(strings.NewReader(htmlStr)) if err != nil { return "", err } diff --git a/webmention_test.go b/webmention_test.go index c49d158..abb7bea 100644 --- a/webmention_test.go +++ b/webmention_test.go @@ -1,7 +1,10 @@ package owl_test import ( + "bytes" "h4kor/owl-blogs" + "io" + "net/http" "testing" ) @@ -12,7 +15,7 @@ import ( func TestParseValidHEntry(t *testing.T) { html := []byte("
Foo
") parser := &owl.OwlHtmlParser{} - entry, err := parser.ParseHEntry(html) + entry, err := parser.ParseHEntry(&http.Response{Body: io.NopCloser(bytes.NewReader(html))}) if err != nil { t.Errorf("Unable to parse feed: %v", err) @@ -25,7 +28,7 @@ func TestParseValidHEntry(t *testing.T) { func TestParseValidHEntryWithoutTitle(t *testing.T) { html := []byte("
Foo
") parser := &owl.OwlHtmlParser{} - entry, err := parser.ParseHEntry(html) + entry, err := parser.ParseHEntry(&http.Response{Body: io.NopCloser(bytes.NewReader(html))}) if err != nil { t.Errorf("Unable to parse feed: %v", err) @@ -38,7 +41,7 @@ func TestParseValidHEntryWithoutTitle(t *testing.T) { func TestGetWebmentionEndpointLink(t *testing.T) { html := []byte("") parser := &owl.OwlHtmlParser{} - endpoint, err := parser.GetWebmentionEndpoint(html) + endpoint, err := parser.GetWebmentionEndpoint(&http.Response{Body: io.NopCloser(bytes.NewReader(html))}) if err != nil { t.Errorf("Unable to parse feed: %v", err) @@ -51,7 +54,7 @@ func TestGetWebmentionEndpointLink(t *testing.T) { func TestGetWebmentionEndpointLinkA(t *testing.T) { html := []byte("") parser := &owl.OwlHtmlParser{} - endpoint, err := parser.GetWebmentionEndpoint(html) + endpoint, err := parser.GetWebmentionEndpoint(&http.Response{Body: io.NopCloser(bytes.NewReader(html))}) if err != nil { t.Errorf("Unable to parse feed: %v", err) @@ -60,3 +63,43 @@ func TestGetWebmentionEndpointLinkA(t *testing.T) { t.Errorf("Wrong endpoint. Expected %v, got %v", "http://example.com/webmention", endpoint) } } + +// func TestRealWorldWebmention(t *testing.T) { +// links := []string{ +// "https://webmention.rocks/test/1", +// "https://webmention.rocks/test/2", +// "https://webmention.rocks/test/3", +// "https://webmention.rocks/test/4", +// "https://webmention.rocks/test/5", +// "https://webmention.rocks/test/6", +// "https://webmention.rocks/test/7", +// "https://webmention.rocks/test/8", +// "https://webmention.rocks/test/9", +// "https://webmention.rocks/test/10", +// "https://webmention.rocks/test/11", +// "https://webmention.rocks/test/12", +// "https://webmention.rocks/test/13", +// "https://webmention.rocks/test/14", +// "https://webmention.rocks/test/15", +// "https://webmention.rocks/test/16", +// "https://webmention.rocks/test/17", +// "https://webmention.rocks/test/18", +// "https://webmention.rocks/test/19", +// "https://webmention.rocks/test/20", +// "https://webmention.rocks/test/21", +// "https://webmention.rocks/test/22", +// "https://webmention.rocks/test/23/page", +// } + +// for _, link := range links { +// parser := &owl.OwlHtmlParser{} +// client := &owl.OwlHttpClient{} +// html, _ := client.Get(link) +// _, err := parser.GetWebmentionEndpoint(html) + +// if err != nil { +// t.Errorf("Unable to find webmention: %v for link %v", err, link) +// } +// } + +// }