diff --git a/README.md b/README.md index f9e3993..e4afd22 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,9 @@ Each directory in the `/users/` directory of a repository is considered a user. -- This will be rendered as the blog post. -- Must be present for the blog post to be valid. -- All other folders will be ignored + \- status.yml + -- Used to track various process status related to the post, + -- such as if a webmention was sent. \- media/ -- Contains all media files used in the blog post. -- All files in this folder will be publicly available @@ -59,3 +62,14 @@ aliases: Actual post ``` + + +#### status.yml + +``` +webmentions: + - target: https://example.com/post + supported: true + scanned_at: 2021-08-13T17:07:00Z + last_sent_at: 2021-08-13T17:07:00Z +``` \ No newline at end of file diff --git a/owl_test.go b/owl_test.go index 5d3b5f4..3523afc 100644 --- a/owl_test.go +++ b/owl_test.go @@ -3,21 +3,34 @@ package owl_test import ( "h4kor/owl-blogs" "math/rand" + "net/url" "time" ) -type MockMicroformatParser struct{} +type MockHttpParser struct{} -func (*MockMicroformatParser) ParseHEntry(data []byte) (owl.ParsedHEntry, error) { +func (*MockHttpParser) ParseHEntry(data []byte) (owl.ParsedHEntry, error) { return owl.ParsedHEntry{Title: "Mock Title"}, nil } +func (*MockHttpParser) ParseLinks(data []byte) ([]string, error) { + return []string{"http://example.com"}, nil +} + +func (*MockHttpParser) GetWebmentionEndpoint(data []byte) (string, error) { + return "http://example.com/webmention", nil +} + type MockHttpRetriever struct{} func (*MockHttpRetriever) Get(url string) ([]byte, error) { return []byte(""), nil } +func (m *MockHttpRetriever) Post(url string, data url.Values) ([]byte, error) { + return []byte(""), nil +} + func randomName() string { rand.Seed(time.Now().UnixNano()) var letters = []rune("abcdefghijklmnopqrstuvwxyz") diff --git a/post.go b/post.go index f2a6ff9..d9fabd4 100644 --- a/post.go +++ b/post.go @@ -6,9 +6,11 @@ import ( "encoding/base64" "fmt" "io/ioutil" + "net/url" "os" "path" "sort" + "time" "github.com/yuin/goldmark" "github.com/yuin/goldmark/extension" @@ -32,6 +34,10 @@ type PostMeta struct { Draft bool `yaml:"draft"` } +type PostStatus struct { + Webmentions []WebmentionOut +} + func (post Post) Id() string { return post.id } @@ -40,6 +46,10 @@ func (post Post) Dir() string { return path.Join(post.user.Dir(), "public", post.id) } +func (post Post) StatusFile() string { + return path.Join(post.Dir(), "status.yml") +} + func (post Post) MediaDir() string { return path.Join(post.Dir(), "media") } @@ -81,6 +91,42 @@ func (post Post) Content() []byte { return data } +func (post Post) Status() PostStatus { + // read status file + // return parsed webmentions + fileName := post.StatusFile() + if !fileExists(fileName) { + return PostStatus{} + } + + data, err := os.ReadFile(fileName) + if err != nil { + return PostStatus{} + } + + status := PostStatus{} + err = yaml.Unmarshal(data, &status) + if err != nil { + return PostStatus{} + } + + return status +} + +func (post Post) PersistStatus(status PostStatus) error { + data, err := yaml.Marshal(status) + if err != nil { + return err + } + + err = os.WriteFile(post.StatusFile(), data, 0644) + if err != nil { + return err + } + + return nil +} + func (post Post) RenderedContent() bytes.Buffer { data := post.Content() @@ -156,7 +202,7 @@ func (post *Post) WebmentionFile(source string) string { return path.Join(post.WebmentionDir(), hashStr+".yml") } -func (post *Post) PersistWebmention(webmention Webmention) error { +func (post *Post) PersistWebmention(webmention WebmentionIn) error { // ensure dir exists os.MkdirAll(post.WebmentionDir(), 0755) @@ -169,7 +215,7 @@ func (post *Post) PersistWebmention(webmention Webmention) error { return os.WriteFile(fileName, data, 0644) } -func (post *Post) Webmention(source string) (Webmention, error) { +func (post *Post) Webmention(source string) (WebmentionIn, error) { // ensure dir exists os.MkdirAll(post.WebmentionDir(), 0755) @@ -177,18 +223,18 @@ func (post *Post) Webmention(source string) (Webmention, error) { fileName := post.WebmentionFile(source) if !fileExists(fileName) { // return error if file doesn't exist - return Webmention{}, fmt.Errorf("Webmention file not found: %s", source) + return WebmentionIn{}, fmt.Errorf("Webmention file not found: %s", source) } data, err := os.ReadFile(fileName) if err != nil { - return Webmention{}, err + return WebmentionIn{}, err } - mention := Webmention{} + mention := WebmentionIn{} err = yaml.Unmarshal(data, &mention) if err != nil { - return Webmention{}, err + return WebmentionIn{}, err } return mention, nil @@ -198,7 +244,7 @@ func (post *Post) AddWebmention(source string) error { // Check if file already exists _, err := post.Webmention(source) if err != nil { - webmention := Webmention{ + webmention := WebmentionIn{ Source: source, } defer post.EnrichWebmention(source) @@ -207,8 +253,49 @@ func (post *Post) AddWebmention(source string) error { return nil } +func (post *Post) AddOutgoingWebmention(target string) error { + status := post.Status() + + // Check if file already exists + _, err := post.Webmention(target) + if err != nil { + webmention := WebmentionOut{ + Target: target, + } + // if target is not in status, add it + for _, t := range status.Webmentions { + if t.Target == webmention.Target { + return nil + } + } + status.Webmentions = append(status.Webmentions, webmention) + } + + return post.PersistStatus(status) +} + +func (post *Post) UpdateOutgoingWebmention(webmention *WebmentionOut) error { + status := post.Status() + + // if target is not in status, add it + replaced := false + for i, t := range status.Webmentions { + if t.Target == webmention.Target { + status.Webmentions[i] = *webmention + replaced = true + break + } + } + + if !replaced { + status.Webmentions = append(status.Webmentions, *webmention) + } + + return post.PersistStatus(status) +} + func (post *Post) EnrichWebmention(source string) error { - html, err := post.user.repo.Retriever.Get(source) + html, err := post.user.repo.HttpClient.Get(source) if err == nil { webmention, err := post.Webmention(source) if err != nil { @@ -223,17 +310,17 @@ func (post *Post) EnrichWebmention(source string) error { return err } -func (post *Post) Webmentions() []Webmention { +func (post *Post) Webmentions() []WebmentionIn { // ensure dir exists os.MkdirAll(post.WebmentionDir(), 0755) files := listDir(post.WebmentionDir()) - webmentions := []Webmention{} + webmentions := []WebmentionIn{} for _, file := range files { data, err := os.ReadFile(path.Join(post.WebmentionDir(), file)) if err != nil { continue } - mention := Webmention{} + mention := WebmentionIn{} err = yaml.Unmarshal(data, &mention) if err != nil { continue @@ -244,9 +331,9 @@ func (post *Post) Webmentions() []Webmention { return webmentions } -func (post *Post) ApprovedWebmentions() []Webmention { +func (post *Post) ApprovedWebmentions() []WebmentionIn { webmentions := post.Webmentions() - approved := []Webmention{} + approved := []WebmentionIn{} for _, webmention := range webmentions { if webmention.ApprovalStatus == "approved" { approved = append(approved, webmention) @@ -259,3 +346,59 @@ func (post *Post) ApprovedWebmentions() []Webmention { }) return approved } + +func (post *Post) OutgoingWebmentions() []WebmentionOut { + status := post.Status() + return status.Webmentions + +} + +// ScanForLinks scans the post content for links and adds them to the +// `status.yml` file for the post. The links are not scanned by this function. +func (post *Post) ScanForLinks() error { + // this could be done in markdown parsing, but I don't want to + // rely on goldmark for this (yet) + postHtml, err := renderPostContent(post) + if err != nil { + return err + } + links, _ := post.user.repo.Parser.ParseLinks([]byte(postHtml)) + for _, link := range links { + post.AddOutgoingWebmention(link) + } + return nil +} + +func (post *Post) SendWebmention(webmention WebmentionOut) error { + defer post.UpdateOutgoingWebmention(&webmention) + webmention.ScannedAt = time.Now() + + html, err := post.user.repo.HttpClient.Get(webmention.Target) + if err != nil { + // TODO handle error + webmention.Supported = false + return err + } + endpoint, err := post.user.repo.Parser.GetWebmentionEndpoint(html) + if err != nil { + // TODO handle error + webmention.Supported = false + return err + } + webmention.Supported = true + + // send webmention + payload := url.Values{} + payload.Set("source", post.FullUrl()) + payload.Set("target", webmention.Target) + _, err = post.user.repo.HttpClient.Post(endpoint, payload) + + if err != nil { + // TODO handle error + return err + } + + // update webmention status + webmention.LastSentAt = time.Now() + return nil +} diff --git a/post_test.go b/post_test.go index 5eafb59..6c441e2 100644 --- a/post_test.go +++ b/post_test.go @@ -173,7 +173,7 @@ func TestPersistWebmention(t *testing.T) { repo := getTestRepo() user, _ := repo.CreateUser("testuser") post, _ := user.CreateNewPost("testpost") - webmention := owl.Webmention{ + webmention := owl.WebmentionIn{ Source: "http://example.com/source", } err := post.PersistWebmention(webmention) @@ -192,8 +192,8 @@ func TestPersistWebmention(t *testing.T) { func TestAddWebmentionCreatesFile(t *testing.T) { repo := getTestRepo() - repo.Retriever = &MockHttpRetriever{} - repo.Parser = &MockMicroformatParser{} + repo.HttpClient = &MockHttpRetriever{} + repo.Parser = &MockHttpParser{} user, _ := repo.CreateUser("testuser") post, _ := user.CreateNewPost("testpost") @@ -210,8 +210,8 @@ func TestAddWebmentionCreatesFile(t *testing.T) { func TestAddWebmentionNotOverwritingFile(t *testing.T) { repo := getTestRepo() - repo.Retriever = &MockHttpRetriever{} - repo.Parser = &MockMicroformatParser{} + repo.HttpClient = &MockHttpRetriever{} + repo.Parser = &MockHttpParser{} user, _ := repo.CreateUser("testuser") post, _ := user.CreateNewPost("testpost") @@ -240,8 +240,8 @@ func TestAddWebmentionNotOverwritingFile(t *testing.T) { func TestAddWebmentionAddsParsedTitle(t *testing.T) { repo := getTestRepo() - repo.Retriever = &MockHttpRetriever{} - repo.Parser = &MockMicroformatParser{} + repo.HttpClient = &MockHttpRetriever{} + repo.Parser = &MockHttpParser{} user, _ := repo.CreateUser("testuser") post, _ := user.CreateNewPost("testpost") @@ -265,25 +265,25 @@ func TestApprovedWebmentions(t *testing.T) { repo := getTestRepo() user, _ := repo.CreateUser("testuser") post, _ := user.CreateNewPost("testpost") - webmention := owl.Webmention{ + webmention := owl.WebmentionIn{ Source: "http://example.com/source", ApprovalStatus: "approved", RetrievedAt: time.Now(), } post.PersistWebmention(webmention) - webmention = owl.Webmention{ + webmention = owl.WebmentionIn{ Source: "http://example.com/source2", ApprovalStatus: "", RetrievedAt: time.Now().Add(time.Hour * -1), } post.PersistWebmention(webmention) - webmention = owl.Webmention{ + webmention = owl.WebmentionIn{ Source: "http://example.com/source3", ApprovalStatus: "approved", RetrievedAt: time.Now().Add(time.Hour * -2), } post.PersistWebmention(webmention) - webmention = owl.Webmention{ + webmention = owl.WebmentionIn{ Source: "http://example.com/source4", ApprovalStatus: "rejected", RetrievedAt: time.Now().Add(time.Hour * -3), @@ -303,3 +303,83 @@ func TestApprovedWebmentions(t *testing.T) { } } + +func TestScanningForLinks(t *testing.T) { + repo := getTestRepo() + user, _ := repo.CreateUser("testuser") + post, _ := user.CreateNewPost("testpost") + + content := "---\n" + content += "title: test\n" + content += "date: Wed, 17 Aug 2022 10:50:02 +0000\n" + content += "---\n" + content += "\n" + content += "[Hello](https://example.com/hello)\n" + os.WriteFile(post.ContentFile(), []byte(content), 0644) + + post.ScanForLinks() + webmentions := post.OutgoingWebmentions() + if len(webmentions) != 1 { + t.Errorf("Expected 1 webmention, got %d", len(webmentions)) + } + if webmentions[0].Target != "https://example.com/hello" { + t.Errorf("Expected target: %s, got %s", "https://example.com/hello", webmentions[0].Target) + } +} + +func TestScanningForLinksDoesNotAddDuplicates(t *testing.T) { + repo := getTestRepo() + user, _ := repo.CreateUser("testuser") + post, _ := user.CreateNewPost("testpost") + + content := "---\n" + content += "title: test\n" + content += "date: Wed, 17 Aug 2022 10:50:02 +0000\n" + content += "---\n" + content += "\n" + content += "[Hello](https://example.com/hello)\n" + content += "[Hello](https://example.com/hello)\n" + os.WriteFile(post.ContentFile(), []byte(content), 0644) + + post.ScanForLinks() + post.ScanForLinks() + post.ScanForLinks() + webmentions := post.OutgoingWebmentions() + if len(webmentions) != 1 { + t.Errorf("Expected 1 webmention, got %d", len(webmentions)) + } + if webmentions[0].Target != "https://example.com/hello" { + t.Errorf("Expected target: %s, got %s", "https://example.com/hello", webmentions[0].Target) + } +} + +func TestCanSendWebmention(t *testing.T) { + repo := getTestRepo() + repo.HttpClient = &MockHttpRetriever{} + repo.Parser = &MockHttpParser{} + user, _ := repo.CreateUser("testuser") + post, _ := user.CreateNewPost("testpost") + + webmention := owl.WebmentionOut{ + Target: "http://example.com", + } + + err := post.SendWebmention(webmention) + if err != nil { + t.Errorf("Error sending webmention: %v", err) + } + + webmentions := post.OutgoingWebmentions() + + if len(webmentions) != 1 { + t.Errorf("Expected 1 webmention, got %d", len(webmentions)) + } + + if webmentions[0].Target != "http://example.com" { + t.Errorf("Expected target: %s, got %s", "http://example.com", webmentions[0].Target) + } + + if webmentions[0].LastSentAt.IsZero() { + t.Errorf("Expected LastSentAt to be set") + } +} diff --git a/renderer.go b/renderer.go index 74034a6..a1b7643 100644 --- a/renderer.go +++ b/renderer.go @@ -68,13 +68,18 @@ func renderIntoBaseTemplate(user User, data PageContent) (string, error) { return html.String(), nil } -func RenderPost(post *Post) (string, error) { +func renderPostContent(post *Post) (string, error) { buf := post.RenderedContent() postHtml, err := renderEmbedTemplate("embed/post.html", PostRenderData{ Title: post.Title(), Post: post, Content: template.HTML(buf.String()), }) + return postHtml, err +} + +func RenderPost(post *Post) (string, error) { + postHtml, err := renderPostContent(post) if err != nil { return "", err } diff --git a/renderer_test.go b/renderer_test.go index c6925fa..92d9cd7 100644 --- a/renderer_test.go +++ b/renderer_test.go @@ -162,14 +162,14 @@ func TestRenderPostIncludesRelToWebMention(t *testing.T) { func TestRenderPostAddsLinksToApprovedWebmention(t *testing.T) { user := getTestUser() post, _ := user.CreateNewPost("testpost") - webmention := owl.Webmention{ + webmention := owl.WebmentionIn{ Source: "http://example.com/source3", Title: "Test Title", ApprovalStatus: "approved", RetrievedAt: time.Now().Add(time.Hour * -2), } post.PersistWebmention(webmention) - webmention = owl.Webmention{ + webmention = owl.WebmentionIn{ Source: "http://example.com/source4", ApprovalStatus: "rejected", RetrievedAt: time.Now().Add(time.Hour * -3), diff --git a/repository.go b/repository.go index c3c8c66..c1d1759 100644 --- a/repository.go +++ b/repository.go @@ -20,8 +20,8 @@ type Repository struct { single_user_mode bool active_user string allow_raw_html bool - Retriever HttpRetriever - Parser MicroformatParser + HttpClient HttpClient + Parser HtmlParser } type RepoConfig struct { @@ -29,7 +29,7 @@ type RepoConfig struct { } func CreateRepository(name string) (Repository, error) { - newRepo := Repository{name: name, Parser: OwlMicroformatParser{}, Retriever: OwlHttpRetriever{}} + newRepo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}} // check if repository already exists if dirExists(newRepo.Dir()) { return Repository{}, fmt.Errorf("Repository already exists") @@ -63,7 +63,7 @@ func CreateRepository(name string) (Repository, error) { func OpenRepository(name string) (Repository, error) { - repo := Repository{name: name, Parser: OwlMicroformatParser{}, Retriever: OwlHttpRetriever{}} + repo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}} if !dirExists(repo.Dir()) { return Repository{}, fmt.Errorf("Repository does not exist: " + repo.Dir()) } diff --git a/webmention.go b/webmention.go index fccd815..df4ab33 100644 --- a/webmention.go +++ b/webmention.go @@ -4,36 +4,47 @@ import ( "bytes" "errors" "net/http" + "net/url" "strings" "time" "golang.org/x/net/html" ) -type Webmention struct { +type WebmentionIn struct { Source string `yaml:"source"` Title string `yaml:"title"` ApprovalStatus string `yaml:"approval_status"` RetrievedAt time.Time `yaml:"retrieved_at"` } -type HttpRetriever interface { +type WebmentionOut struct { + Target string `yaml:"target"` + Supported bool `yaml:"supported"` + ScannedAt time.Time `yaml:"scanned_at"` + LastSentAt time.Time `yaml:"last_sent_at"` +} + +type HttpClient interface { Get(url string) ([]byte, error) + Post(url string, data url.Values) ([]byte, error) } -type MicroformatParser interface { +type HtmlParser interface { ParseHEntry(data []byte) (ParsedHEntry, error) + ParseLinks(data []byte) ([]string, error) + GetWebmentionEndpoint(data []byte) (string, error) } -type OwlHttpRetriever struct{} +type OwlHttpClient struct{} -type OwlMicroformatParser struct{} +type OwlHtmlParser struct{} type ParsedHEntry struct { Title string } -func (OwlHttpRetriever) Get(url string) ([]byte, error) { +func (OwlHttpClient) Get(url string) ([]byte, error) { resp, err := http.Get(url) if err != nil { return []byte{}, err @@ -44,6 +55,17 @@ func (OwlHttpRetriever) Get(url string) ([]byte, error) { return data, err } +func (OwlHttpClient) Post(url string, data url.Values) ([]byte, error) { + resp, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) + if err != nil { + return []byte{}, err + } + var respData []byte + _, err = resp.Body.Read(respData) + + return respData, err +} + func collectText(n *html.Node, buf *bytes.Buffer) { if n.Type == html.TextNode { buf.WriteString(n.Data) @@ -53,7 +75,7 @@ func collectText(n *html.Node, buf *bytes.Buffer) { } } -func (OwlMicroformatParser) ParseHEntry(data []byte) (ParsedHEntry, error) { +func (OwlHtmlParser) ParseHEntry(data []byte) (ParsedHEntry, error) { doc, err := html.Parse(strings.NewReader(string(data))) if err != nil { return ParsedHEntry{}, err @@ -95,3 +117,59 @@ func (OwlMicroformatParser) ParseHEntry(data []byte) (ParsedHEntry, error) { } return findHFeed(doc) } + +func (OwlHtmlParser) ParseLinks(data []byte) ([]string, error) { + doc, err := html.Parse(strings.NewReader(string(data))) + if err != nil { + return make([]string, 0), err + } + + var findLinks func(*html.Node) ([]string, error) + findLinks = func(n *html.Node) ([]string, error) { + links := make([]string, 0) + if n.Type == html.ElementNode && n.Data == "a" { + for _, attr := range n.Attr { + if attr.Key == "href" { + links = append(links, attr.Val) + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + childLinks, _ := findLinks(c) + links = append(links, childLinks...) + } + return links, nil + } + return findLinks(doc) + +} + +func (OwlHtmlParser) GetWebmentionEndpoint(data []byte) (string, error) { + doc, err := html.Parse(strings.NewReader(string(data))) + if err != nil { + return "", err + } + + var findEndpoint func(*html.Node) (string, error) + findEndpoint = func(n *html.Node) (string, error) { + if n.Type == html.ElementNode && n.Data == "link" { + for _, attr := range n.Attr { + if attr.Key == "rel" && attr.Val == "webmention" { + for _, attr := range n.Attr { + if attr.Key == "href" { + return attr.Val, nil + } + } + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + endpoint, err := findEndpoint(c) + if err == nil { + return endpoint, nil + } + } + return "", errors.New("no webmention endpoint found") + } + return findEndpoint(doc) +} diff --git a/webmention_test.go b/webmention_test.go index e02d9cf..c7e95c7 100644 --- a/webmention_test.go +++ b/webmention_test.go @@ -11,7 +11,7 @@ import ( func TestParseValidHEntry(t *testing.T) { html := []byte("
Foo
") - parser := &owl.OwlMicroformatParser{} + parser := &owl.OwlHtmlParser{} entry, err := parser.ParseHEntry(html) if err != nil { @@ -24,7 +24,7 @@ func TestParseValidHEntry(t *testing.T) { func TestParseValidHEntryWithoutTitle(t *testing.T) { html := []byte("
Foo
") - parser := &owl.OwlMicroformatParser{} + parser := &owl.OwlHtmlParser{} entry, err := parser.ParseHEntry(html) if err != nil {