Merge branch 'webmention' of ssh://git.libove.org:222/h4kor/owl-blogs into webmention

This commit is contained in:
Niko Abeler 2022-09-05 20:34:43 +02:00
commit 2614ee3f15
9 changed files with 375 additions and 42 deletions

View File

@ -23,6 +23,9 @@ Each directory in the `/users/` directory of a repository is considered a user.
-- This will be rendered as the blog post.
-- Must be present for the blog post to be valid.
-- All other folders will be ignored
\- status.yml
-- Used to track various process status related to the post,
-- such as if a webmention was sent.
\- media/
-- Contains all media files used in the blog post.
-- All files in this folder will be publicly available
@ -59,3 +62,14 @@ aliases:
Actual post
```
#### status.yml
```
webmentions:
- target: https://example.com/post
supported: true
scanned_at: 2021-08-13T17:07:00Z
last_sent_at: 2021-08-13T17:07:00Z
```

View File

@ -3,21 +3,34 @@ package owl_test
import (
"h4kor/owl-blogs"
"math/rand"
"net/url"
"time"
)
type MockMicroformatParser struct{}
type MockHttpParser struct{}
func (*MockMicroformatParser) ParseHEntry(data []byte) (owl.ParsedHEntry, error) {
func (*MockHttpParser) ParseHEntry(data []byte) (owl.ParsedHEntry, error) {
return owl.ParsedHEntry{Title: "Mock Title"}, nil
}
func (*MockHttpParser) ParseLinks(data []byte) ([]string, error) {
return []string{"http://example.com"}, nil
}
func (*MockHttpParser) GetWebmentionEndpoint(data []byte) (string, error) {
return "http://example.com/webmention", nil
}
type MockHttpRetriever struct{}
func (*MockHttpRetriever) Get(url string) ([]byte, error) {
return []byte(""), nil
}
func (m *MockHttpRetriever) Post(url string, data url.Values) ([]byte, error) {
return []byte(""), nil
}
func randomName() string {
rand.Seed(time.Now().UnixNano())
var letters = []rune("abcdefghijklmnopqrstuvwxyz")

169
post.go
View File

@ -6,9 +6,11 @@ import (
"encoding/base64"
"fmt"
"io/ioutil"
"net/url"
"os"
"path"
"sort"
"time"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
@ -32,6 +34,10 @@ type PostMeta struct {
Draft bool `yaml:"draft"`
}
type PostStatus struct {
Webmentions []WebmentionOut
}
func (post Post) Id() string {
return post.id
}
@ -40,6 +46,10 @@ func (post Post) Dir() string {
return path.Join(post.user.Dir(), "public", post.id)
}
func (post Post) StatusFile() string {
return path.Join(post.Dir(), "status.yml")
}
func (post Post) MediaDir() string {
return path.Join(post.Dir(), "media")
}
@ -81,6 +91,42 @@ func (post Post) Content() []byte {
return data
}
func (post Post) Status() PostStatus {
// read status file
// return parsed webmentions
fileName := post.StatusFile()
if !fileExists(fileName) {
return PostStatus{}
}
data, err := os.ReadFile(fileName)
if err != nil {
return PostStatus{}
}
status := PostStatus{}
err = yaml.Unmarshal(data, &status)
if err != nil {
return PostStatus{}
}
return status
}
func (post Post) PersistStatus(status PostStatus) error {
data, err := yaml.Marshal(status)
if err != nil {
return err
}
err = os.WriteFile(post.StatusFile(), data, 0644)
if err != nil {
return err
}
return nil
}
func (post Post) RenderedContent() bytes.Buffer {
data := post.Content()
@ -156,7 +202,7 @@ func (post *Post) WebmentionFile(source string) string {
return path.Join(post.WebmentionDir(), hashStr+".yml")
}
func (post *Post) PersistWebmention(webmention Webmention) error {
func (post *Post) PersistWebmention(webmention WebmentionIn) error {
// ensure dir exists
os.MkdirAll(post.WebmentionDir(), 0755)
@ -169,7 +215,7 @@ func (post *Post) PersistWebmention(webmention Webmention) error {
return os.WriteFile(fileName, data, 0644)
}
func (post *Post) Webmention(source string) (Webmention, error) {
func (post *Post) Webmention(source string) (WebmentionIn, error) {
// ensure dir exists
os.MkdirAll(post.WebmentionDir(), 0755)
@ -177,18 +223,18 @@ func (post *Post) Webmention(source string) (Webmention, error) {
fileName := post.WebmentionFile(source)
if !fileExists(fileName) {
// return error if file doesn't exist
return Webmention{}, fmt.Errorf("Webmention file not found: %s", source)
return WebmentionIn{}, fmt.Errorf("Webmention file not found: %s", source)
}
data, err := os.ReadFile(fileName)
if err != nil {
return Webmention{}, err
return WebmentionIn{}, err
}
mention := Webmention{}
mention := WebmentionIn{}
err = yaml.Unmarshal(data, &mention)
if err != nil {
return Webmention{}, err
return WebmentionIn{}, err
}
return mention, nil
@ -198,7 +244,7 @@ func (post *Post) AddWebmention(source string) error {
// Check if file already exists
_, err := post.Webmention(source)
if err != nil {
webmention := Webmention{
webmention := WebmentionIn{
Source: source,
}
defer post.EnrichWebmention(source)
@ -207,8 +253,49 @@ func (post *Post) AddWebmention(source string) error {
return nil
}
func (post *Post) AddOutgoingWebmention(target string) error {
status := post.Status()
// Check if file already exists
_, err := post.Webmention(target)
if err != nil {
webmention := WebmentionOut{
Target: target,
}
// if target is not in status, add it
for _, t := range status.Webmentions {
if t.Target == webmention.Target {
return nil
}
}
status.Webmentions = append(status.Webmentions, webmention)
}
return post.PersistStatus(status)
}
func (post *Post) UpdateOutgoingWebmention(webmention *WebmentionOut) error {
status := post.Status()
// if target is not in status, add it
replaced := false
for i, t := range status.Webmentions {
if t.Target == webmention.Target {
status.Webmentions[i] = *webmention
replaced = true
break
}
}
if !replaced {
status.Webmentions = append(status.Webmentions, *webmention)
}
return post.PersistStatus(status)
}
func (post *Post) EnrichWebmention(source string) error {
html, err := post.user.repo.Retriever.Get(source)
html, err := post.user.repo.HttpClient.Get(source)
if err == nil {
webmention, err := post.Webmention(source)
if err != nil {
@ -223,17 +310,17 @@ func (post *Post) EnrichWebmention(source string) error {
return err
}
func (post *Post) Webmentions() []Webmention {
func (post *Post) Webmentions() []WebmentionIn {
// ensure dir exists
os.MkdirAll(post.WebmentionDir(), 0755)
files := listDir(post.WebmentionDir())
webmentions := []Webmention{}
webmentions := []WebmentionIn{}
for _, file := range files {
data, err := os.ReadFile(path.Join(post.WebmentionDir(), file))
if err != nil {
continue
}
mention := Webmention{}
mention := WebmentionIn{}
err = yaml.Unmarshal(data, &mention)
if err != nil {
continue
@ -244,9 +331,9 @@ func (post *Post) Webmentions() []Webmention {
return webmentions
}
func (post *Post) ApprovedWebmentions() []Webmention {
func (post *Post) ApprovedWebmentions() []WebmentionIn {
webmentions := post.Webmentions()
approved := []Webmention{}
approved := []WebmentionIn{}
for _, webmention := range webmentions {
if webmention.ApprovalStatus == "approved" {
approved = append(approved, webmention)
@ -259,3 +346,59 @@ func (post *Post) ApprovedWebmentions() []Webmention {
})
return approved
}
func (post *Post) OutgoingWebmentions() []WebmentionOut {
status := post.Status()
return status.Webmentions
}
// ScanForLinks scans the post content for links and adds them to the
// `status.yml` file for the post. The links are not scanned by this function.
func (post *Post) ScanForLinks() error {
// this could be done in markdown parsing, but I don't want to
// rely on goldmark for this (yet)
postHtml, err := renderPostContent(post)
if err != nil {
return err
}
links, _ := post.user.repo.Parser.ParseLinks([]byte(postHtml))
for _, link := range links {
post.AddOutgoingWebmention(link)
}
return nil
}
func (post *Post) SendWebmention(webmention WebmentionOut) error {
defer post.UpdateOutgoingWebmention(&webmention)
webmention.ScannedAt = time.Now()
html, err := post.user.repo.HttpClient.Get(webmention.Target)
if err != nil {
// TODO handle error
webmention.Supported = false
return err
}
endpoint, err := post.user.repo.Parser.GetWebmentionEndpoint(html)
if err != nil {
// TODO handle error
webmention.Supported = false
return err
}
webmention.Supported = true
// send webmention
payload := url.Values{}
payload.Set("source", post.FullUrl())
payload.Set("target", webmention.Target)
_, err = post.user.repo.HttpClient.Post(endpoint, payload)
if err != nil {
// TODO handle error
return err
}
// update webmention status
webmention.LastSentAt = time.Now()
return nil
}

View File

@ -173,7 +173,7 @@ func TestPersistWebmention(t *testing.T) {
repo := getTestRepo()
user, _ := repo.CreateUser("testuser")
post, _ := user.CreateNewPost("testpost")
webmention := owl.Webmention{
webmention := owl.WebmentionIn{
Source: "http://example.com/source",
}
err := post.PersistWebmention(webmention)
@ -192,8 +192,8 @@ func TestPersistWebmention(t *testing.T) {
func TestAddWebmentionCreatesFile(t *testing.T) {
repo := getTestRepo()
repo.Retriever = &MockHttpRetriever{}
repo.Parser = &MockMicroformatParser{}
repo.HttpClient = &MockHttpRetriever{}
repo.Parser = &MockHttpParser{}
user, _ := repo.CreateUser("testuser")
post, _ := user.CreateNewPost("testpost")
@ -210,8 +210,8 @@ func TestAddWebmentionCreatesFile(t *testing.T) {
func TestAddWebmentionNotOverwritingFile(t *testing.T) {
repo := getTestRepo()
repo.Retriever = &MockHttpRetriever{}
repo.Parser = &MockMicroformatParser{}
repo.HttpClient = &MockHttpRetriever{}
repo.Parser = &MockHttpParser{}
user, _ := repo.CreateUser("testuser")
post, _ := user.CreateNewPost("testpost")
@ -240,8 +240,8 @@ func TestAddWebmentionNotOverwritingFile(t *testing.T) {
func TestAddWebmentionAddsParsedTitle(t *testing.T) {
repo := getTestRepo()
repo.Retriever = &MockHttpRetriever{}
repo.Parser = &MockMicroformatParser{}
repo.HttpClient = &MockHttpRetriever{}
repo.Parser = &MockHttpParser{}
user, _ := repo.CreateUser("testuser")
post, _ := user.CreateNewPost("testpost")
@ -265,25 +265,25 @@ func TestApprovedWebmentions(t *testing.T) {
repo := getTestRepo()
user, _ := repo.CreateUser("testuser")
post, _ := user.CreateNewPost("testpost")
webmention := owl.Webmention{
webmention := owl.WebmentionIn{
Source: "http://example.com/source",
ApprovalStatus: "approved",
RetrievedAt: time.Now(),
}
post.PersistWebmention(webmention)
webmention = owl.Webmention{
webmention = owl.WebmentionIn{
Source: "http://example.com/source2",
ApprovalStatus: "",
RetrievedAt: time.Now().Add(time.Hour * -1),
}
post.PersistWebmention(webmention)
webmention = owl.Webmention{
webmention = owl.WebmentionIn{
Source: "http://example.com/source3",
ApprovalStatus: "approved",
RetrievedAt: time.Now().Add(time.Hour * -2),
}
post.PersistWebmention(webmention)
webmention = owl.Webmention{
webmention = owl.WebmentionIn{
Source: "http://example.com/source4",
ApprovalStatus: "rejected",
RetrievedAt: time.Now().Add(time.Hour * -3),
@ -303,3 +303,83 @@ func TestApprovedWebmentions(t *testing.T) {
}
}
func TestScanningForLinks(t *testing.T) {
repo := getTestRepo()
user, _ := repo.CreateUser("testuser")
post, _ := user.CreateNewPost("testpost")
content := "---\n"
content += "title: test\n"
content += "date: Wed, 17 Aug 2022 10:50:02 +0000\n"
content += "---\n"
content += "\n"
content += "[Hello](https://example.com/hello)\n"
os.WriteFile(post.ContentFile(), []byte(content), 0644)
post.ScanForLinks()
webmentions := post.OutgoingWebmentions()
if len(webmentions) != 1 {
t.Errorf("Expected 1 webmention, got %d", len(webmentions))
}
if webmentions[0].Target != "https://example.com/hello" {
t.Errorf("Expected target: %s, got %s", "https://example.com/hello", webmentions[0].Target)
}
}
func TestScanningForLinksDoesNotAddDuplicates(t *testing.T) {
repo := getTestRepo()
user, _ := repo.CreateUser("testuser")
post, _ := user.CreateNewPost("testpost")
content := "---\n"
content += "title: test\n"
content += "date: Wed, 17 Aug 2022 10:50:02 +0000\n"
content += "---\n"
content += "\n"
content += "[Hello](https://example.com/hello)\n"
content += "[Hello](https://example.com/hello)\n"
os.WriteFile(post.ContentFile(), []byte(content), 0644)
post.ScanForLinks()
post.ScanForLinks()
post.ScanForLinks()
webmentions := post.OutgoingWebmentions()
if len(webmentions) != 1 {
t.Errorf("Expected 1 webmention, got %d", len(webmentions))
}
if webmentions[0].Target != "https://example.com/hello" {
t.Errorf("Expected target: %s, got %s", "https://example.com/hello", webmentions[0].Target)
}
}
func TestCanSendWebmention(t *testing.T) {
repo := getTestRepo()
repo.HttpClient = &MockHttpRetriever{}
repo.Parser = &MockHttpParser{}
user, _ := repo.CreateUser("testuser")
post, _ := user.CreateNewPost("testpost")
webmention := owl.WebmentionOut{
Target: "http://example.com",
}
err := post.SendWebmention(webmention)
if err != nil {
t.Errorf("Error sending webmention: %v", err)
}
webmentions := post.OutgoingWebmentions()
if len(webmentions) != 1 {
t.Errorf("Expected 1 webmention, got %d", len(webmentions))
}
if webmentions[0].Target != "http://example.com" {
t.Errorf("Expected target: %s, got %s", "http://example.com", webmentions[0].Target)
}
if webmentions[0].LastSentAt.IsZero() {
t.Errorf("Expected LastSentAt to be set")
}
}

View File

@ -68,13 +68,18 @@ func renderIntoBaseTemplate(user User, data PageContent) (string, error) {
return html.String(), nil
}
func RenderPost(post *Post) (string, error) {
func renderPostContent(post *Post) (string, error) {
buf := post.RenderedContent()
postHtml, err := renderEmbedTemplate("embed/post.html", PostRenderData{
Title: post.Title(),
Post: post,
Content: template.HTML(buf.String()),
})
return postHtml, err
}
func RenderPost(post *Post) (string, error) {
postHtml, err := renderPostContent(post)
if err != nil {
return "", err
}

View File

@ -162,14 +162,14 @@ func TestRenderPostIncludesRelToWebMention(t *testing.T) {
func TestRenderPostAddsLinksToApprovedWebmention(t *testing.T) {
user := getTestUser()
post, _ := user.CreateNewPost("testpost")
webmention := owl.Webmention{
webmention := owl.WebmentionIn{
Source: "http://example.com/source3",
Title: "Test Title",
ApprovalStatus: "approved",
RetrievedAt: time.Now().Add(time.Hour * -2),
}
post.PersistWebmention(webmention)
webmention = owl.Webmention{
webmention = owl.WebmentionIn{
Source: "http://example.com/source4",
ApprovalStatus: "rejected",
RetrievedAt: time.Now().Add(time.Hour * -3),

View File

@ -20,8 +20,8 @@ type Repository struct {
single_user_mode bool
active_user string
allow_raw_html bool
Retriever HttpRetriever
Parser MicroformatParser
HttpClient HttpClient
Parser HtmlParser
}
type RepoConfig struct {
@ -29,7 +29,7 @@ type RepoConfig struct {
}
func CreateRepository(name string) (Repository, error) {
newRepo := Repository{name: name, Parser: OwlMicroformatParser{}, Retriever: OwlHttpRetriever{}}
newRepo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}}
// check if repository already exists
if dirExists(newRepo.Dir()) {
return Repository{}, fmt.Errorf("Repository already exists")
@ -63,7 +63,7 @@ func CreateRepository(name string) (Repository, error) {
func OpenRepository(name string) (Repository, error) {
repo := Repository{name: name, Parser: OwlMicroformatParser{}, Retriever: OwlHttpRetriever{}}
repo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}}
if !dirExists(repo.Dir()) {
return Repository{}, fmt.Errorf("Repository does not exist: " + repo.Dir())
}

View File

@ -4,36 +4,47 @@ import (
"bytes"
"errors"
"net/http"
"net/url"
"strings"
"time"
"golang.org/x/net/html"
)
type Webmention struct {
type WebmentionIn struct {
Source string `yaml:"source"`
Title string `yaml:"title"`
ApprovalStatus string `yaml:"approval_status"`
RetrievedAt time.Time `yaml:"retrieved_at"`
}
type HttpRetriever interface {
type WebmentionOut struct {
Target string `yaml:"target"`
Supported bool `yaml:"supported"`
ScannedAt time.Time `yaml:"scanned_at"`
LastSentAt time.Time `yaml:"last_sent_at"`
}
type HttpClient interface {
Get(url string) ([]byte, error)
Post(url string, data url.Values) ([]byte, error)
}
type MicroformatParser interface {
type HtmlParser interface {
ParseHEntry(data []byte) (ParsedHEntry, error)
ParseLinks(data []byte) ([]string, error)
GetWebmentionEndpoint(data []byte) (string, error)
}
type OwlHttpRetriever struct{}
type OwlHttpClient struct{}
type OwlMicroformatParser struct{}
type OwlHtmlParser struct{}
type ParsedHEntry struct {
Title string
}
func (OwlHttpRetriever) Get(url string) ([]byte, error) {
func (OwlHttpClient) Get(url string) ([]byte, error) {
resp, err := http.Get(url)
if err != nil {
return []byte{}, err
@ -44,6 +55,17 @@ func (OwlHttpRetriever) Get(url string) ([]byte, error) {
return data, err
}
func (OwlHttpClient) Post(url string, data url.Values) ([]byte, error) {
resp, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
if err != nil {
return []byte{}, err
}
var respData []byte
_, err = resp.Body.Read(respData)
return respData, err
}
func collectText(n *html.Node, buf *bytes.Buffer) {
if n.Type == html.TextNode {
buf.WriteString(n.Data)
@ -53,7 +75,7 @@ func collectText(n *html.Node, buf *bytes.Buffer) {
}
}
func (OwlMicroformatParser) ParseHEntry(data []byte) (ParsedHEntry, error) {
func (OwlHtmlParser) ParseHEntry(data []byte) (ParsedHEntry, error) {
doc, err := html.Parse(strings.NewReader(string(data)))
if err != nil {
return ParsedHEntry{}, err
@ -95,3 +117,59 @@ func (OwlMicroformatParser) ParseHEntry(data []byte) (ParsedHEntry, error) {
}
return findHFeed(doc)
}
func (OwlHtmlParser) ParseLinks(data []byte) ([]string, error) {
doc, err := html.Parse(strings.NewReader(string(data)))
if err != nil {
return make([]string, 0), err
}
var findLinks func(*html.Node) ([]string, error)
findLinks = func(n *html.Node) ([]string, error) {
links := make([]string, 0)
if n.Type == html.ElementNode && n.Data == "a" {
for _, attr := range n.Attr {
if attr.Key == "href" {
links = append(links, attr.Val)
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
childLinks, _ := findLinks(c)
links = append(links, childLinks...)
}
return links, nil
}
return findLinks(doc)
}
func (OwlHtmlParser) GetWebmentionEndpoint(data []byte) (string, error) {
doc, err := html.Parse(strings.NewReader(string(data)))
if err != nil {
return "", err
}
var findEndpoint func(*html.Node) (string, error)
findEndpoint = func(n *html.Node) (string, error) {
if n.Type == html.ElementNode && n.Data == "link" {
for _, attr := range n.Attr {
if attr.Key == "rel" && attr.Val == "webmention" {
for _, attr := range n.Attr {
if attr.Key == "href" {
return attr.Val, nil
}
}
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
endpoint, err := findEndpoint(c)
if err == nil {
return endpoint, nil
}
}
return "", errors.New("no webmention endpoint found")
}
return findEndpoint(doc)
}

View File

@ -11,7 +11,7 @@ import (
func TestParseValidHEntry(t *testing.T) {
html := []byte("<div class=\"h-entry\"><div class=\"p-name\">Foo</div></div>")
parser := &owl.OwlMicroformatParser{}
parser := &owl.OwlHtmlParser{}
entry, err := parser.ParseHEntry(html)
if err != nil {
@ -24,7 +24,7 @@ func TestParseValidHEntry(t *testing.T) {
func TestParseValidHEntryWithoutTitle(t *testing.T) {
html := []byte("<div class=\"h-entry\"></div><div class=\"p-name\">Foo</div>")
parser := &owl.OwlMicroformatParser{}
parser := &owl.OwlHtmlParser{}
entry, err := parser.ParseHEntry(html)
if err != nil {