Merge branch 'webmention' of ssh://git.libove.org:222/h4kor/owl-blogs into webmention
This commit is contained in:
commit
2614ee3f15
14
README.md
14
README.md
|
@ -23,6 +23,9 @@ Each directory in the `/users/` directory of a repository is considered a user.
|
|||
-- This will be rendered as the blog post.
|
||||
-- Must be present for the blog post to be valid.
|
||||
-- All other folders will be ignored
|
||||
\- status.yml
|
||||
-- Used to track various process status related to the post,
|
||||
-- such as if a webmention was sent.
|
||||
\- media/
|
||||
-- Contains all media files used in the blog post.
|
||||
-- All files in this folder will be publicly available
|
||||
|
@ -59,3 +62,14 @@ aliases:
|
|||
Actual post
|
||||
|
||||
```
|
||||
|
||||
|
||||
#### status.yml
|
||||
|
||||
```
|
||||
webmentions:
|
||||
- target: https://example.com/post
|
||||
supported: true
|
||||
scanned_at: 2021-08-13T17:07:00Z
|
||||
last_sent_at: 2021-08-13T17:07:00Z
|
||||
```
|
17
owl_test.go
17
owl_test.go
|
@ -3,21 +3,34 @@ package owl_test
|
|||
import (
|
||||
"h4kor/owl-blogs"
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MockMicroformatParser struct{}
|
||||
type MockHttpParser struct{}
|
||||
|
||||
func (*MockMicroformatParser) ParseHEntry(data []byte) (owl.ParsedHEntry, error) {
|
||||
func (*MockHttpParser) ParseHEntry(data []byte) (owl.ParsedHEntry, error) {
|
||||
return owl.ParsedHEntry{Title: "Mock Title"}, nil
|
||||
}
|
||||
|
||||
func (*MockHttpParser) ParseLinks(data []byte) ([]string, error) {
|
||||
return []string{"http://example.com"}, nil
|
||||
}
|
||||
|
||||
func (*MockHttpParser) GetWebmentionEndpoint(data []byte) (string, error) {
|
||||
return "http://example.com/webmention", nil
|
||||
}
|
||||
|
||||
type MockHttpRetriever struct{}
|
||||
|
||||
func (*MockHttpRetriever) Get(url string) ([]byte, error) {
|
||||
return []byte(""), nil
|
||||
}
|
||||
|
||||
func (m *MockHttpRetriever) Post(url string, data url.Values) ([]byte, error) {
|
||||
return []byte(""), nil
|
||||
}
|
||||
|
||||
func randomName() string {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
var letters = []rune("abcdefghijklmnopqrstuvwxyz")
|
||||
|
|
169
post.go
169
post.go
|
@ -6,9 +6,11 @@ import (
|
|||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/yuin/goldmark"
|
||||
"github.com/yuin/goldmark/extension"
|
||||
|
@ -32,6 +34,10 @@ type PostMeta struct {
|
|||
Draft bool `yaml:"draft"`
|
||||
}
|
||||
|
||||
type PostStatus struct {
|
||||
Webmentions []WebmentionOut
|
||||
}
|
||||
|
||||
func (post Post) Id() string {
|
||||
return post.id
|
||||
}
|
||||
|
@ -40,6 +46,10 @@ func (post Post) Dir() string {
|
|||
return path.Join(post.user.Dir(), "public", post.id)
|
||||
}
|
||||
|
||||
func (post Post) StatusFile() string {
|
||||
return path.Join(post.Dir(), "status.yml")
|
||||
}
|
||||
|
||||
func (post Post) MediaDir() string {
|
||||
return path.Join(post.Dir(), "media")
|
||||
}
|
||||
|
@ -81,6 +91,42 @@ func (post Post) Content() []byte {
|
|||
return data
|
||||
}
|
||||
|
||||
func (post Post) Status() PostStatus {
|
||||
// read status file
|
||||
// return parsed webmentions
|
||||
fileName := post.StatusFile()
|
||||
if !fileExists(fileName) {
|
||||
return PostStatus{}
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(fileName)
|
||||
if err != nil {
|
||||
return PostStatus{}
|
||||
}
|
||||
|
||||
status := PostStatus{}
|
||||
err = yaml.Unmarshal(data, &status)
|
||||
if err != nil {
|
||||
return PostStatus{}
|
||||
}
|
||||
|
||||
return status
|
||||
}
|
||||
|
||||
func (post Post) PersistStatus(status PostStatus) error {
|
||||
data, err := yaml.Marshal(status)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = os.WriteFile(post.StatusFile(), data, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (post Post) RenderedContent() bytes.Buffer {
|
||||
data := post.Content()
|
||||
|
||||
|
@ -156,7 +202,7 @@ func (post *Post) WebmentionFile(source string) string {
|
|||
return path.Join(post.WebmentionDir(), hashStr+".yml")
|
||||
}
|
||||
|
||||
func (post *Post) PersistWebmention(webmention Webmention) error {
|
||||
func (post *Post) PersistWebmention(webmention WebmentionIn) error {
|
||||
// ensure dir exists
|
||||
os.MkdirAll(post.WebmentionDir(), 0755)
|
||||
|
||||
|
@ -169,7 +215,7 @@ func (post *Post) PersistWebmention(webmention Webmention) error {
|
|||
return os.WriteFile(fileName, data, 0644)
|
||||
}
|
||||
|
||||
func (post *Post) Webmention(source string) (Webmention, error) {
|
||||
func (post *Post) Webmention(source string) (WebmentionIn, error) {
|
||||
// ensure dir exists
|
||||
os.MkdirAll(post.WebmentionDir(), 0755)
|
||||
|
||||
|
@ -177,18 +223,18 @@ func (post *Post) Webmention(source string) (Webmention, error) {
|
|||
fileName := post.WebmentionFile(source)
|
||||
if !fileExists(fileName) {
|
||||
// return error if file doesn't exist
|
||||
return Webmention{}, fmt.Errorf("Webmention file not found: %s", source)
|
||||
return WebmentionIn{}, fmt.Errorf("Webmention file not found: %s", source)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(fileName)
|
||||
if err != nil {
|
||||
return Webmention{}, err
|
||||
return WebmentionIn{}, err
|
||||
}
|
||||
|
||||
mention := Webmention{}
|
||||
mention := WebmentionIn{}
|
||||
err = yaml.Unmarshal(data, &mention)
|
||||
if err != nil {
|
||||
return Webmention{}, err
|
||||
return WebmentionIn{}, err
|
||||
}
|
||||
|
||||
return mention, nil
|
||||
|
@ -198,7 +244,7 @@ func (post *Post) AddWebmention(source string) error {
|
|||
// Check if file already exists
|
||||
_, err := post.Webmention(source)
|
||||
if err != nil {
|
||||
webmention := Webmention{
|
||||
webmention := WebmentionIn{
|
||||
Source: source,
|
||||
}
|
||||
defer post.EnrichWebmention(source)
|
||||
|
@ -207,8 +253,49 @@ func (post *Post) AddWebmention(source string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (post *Post) AddOutgoingWebmention(target string) error {
|
||||
status := post.Status()
|
||||
|
||||
// Check if file already exists
|
||||
_, err := post.Webmention(target)
|
||||
if err != nil {
|
||||
webmention := WebmentionOut{
|
||||
Target: target,
|
||||
}
|
||||
// if target is not in status, add it
|
||||
for _, t := range status.Webmentions {
|
||||
if t.Target == webmention.Target {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
status.Webmentions = append(status.Webmentions, webmention)
|
||||
}
|
||||
|
||||
return post.PersistStatus(status)
|
||||
}
|
||||
|
||||
func (post *Post) UpdateOutgoingWebmention(webmention *WebmentionOut) error {
|
||||
status := post.Status()
|
||||
|
||||
// if target is not in status, add it
|
||||
replaced := false
|
||||
for i, t := range status.Webmentions {
|
||||
if t.Target == webmention.Target {
|
||||
status.Webmentions[i] = *webmention
|
||||
replaced = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !replaced {
|
||||
status.Webmentions = append(status.Webmentions, *webmention)
|
||||
}
|
||||
|
||||
return post.PersistStatus(status)
|
||||
}
|
||||
|
||||
func (post *Post) EnrichWebmention(source string) error {
|
||||
html, err := post.user.repo.Retriever.Get(source)
|
||||
html, err := post.user.repo.HttpClient.Get(source)
|
||||
if err == nil {
|
||||
webmention, err := post.Webmention(source)
|
||||
if err != nil {
|
||||
|
@ -223,17 +310,17 @@ func (post *Post) EnrichWebmention(source string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func (post *Post) Webmentions() []Webmention {
|
||||
func (post *Post) Webmentions() []WebmentionIn {
|
||||
// ensure dir exists
|
||||
os.MkdirAll(post.WebmentionDir(), 0755)
|
||||
files := listDir(post.WebmentionDir())
|
||||
webmentions := []Webmention{}
|
||||
webmentions := []WebmentionIn{}
|
||||
for _, file := range files {
|
||||
data, err := os.ReadFile(path.Join(post.WebmentionDir(), file))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
mention := Webmention{}
|
||||
mention := WebmentionIn{}
|
||||
err = yaml.Unmarshal(data, &mention)
|
||||
if err != nil {
|
||||
continue
|
||||
|
@ -244,9 +331,9 @@ func (post *Post) Webmentions() []Webmention {
|
|||
return webmentions
|
||||
}
|
||||
|
||||
func (post *Post) ApprovedWebmentions() []Webmention {
|
||||
func (post *Post) ApprovedWebmentions() []WebmentionIn {
|
||||
webmentions := post.Webmentions()
|
||||
approved := []Webmention{}
|
||||
approved := []WebmentionIn{}
|
||||
for _, webmention := range webmentions {
|
||||
if webmention.ApprovalStatus == "approved" {
|
||||
approved = append(approved, webmention)
|
||||
|
@ -259,3 +346,59 @@ func (post *Post) ApprovedWebmentions() []Webmention {
|
|||
})
|
||||
return approved
|
||||
}
|
||||
|
||||
func (post *Post) OutgoingWebmentions() []WebmentionOut {
|
||||
status := post.Status()
|
||||
return status.Webmentions
|
||||
|
||||
}
|
||||
|
||||
// ScanForLinks scans the post content for links and adds them to the
|
||||
// `status.yml` file for the post. The links are not scanned by this function.
|
||||
func (post *Post) ScanForLinks() error {
|
||||
// this could be done in markdown parsing, but I don't want to
|
||||
// rely on goldmark for this (yet)
|
||||
postHtml, err := renderPostContent(post)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
links, _ := post.user.repo.Parser.ParseLinks([]byte(postHtml))
|
||||
for _, link := range links {
|
||||
post.AddOutgoingWebmention(link)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (post *Post) SendWebmention(webmention WebmentionOut) error {
|
||||
defer post.UpdateOutgoingWebmention(&webmention)
|
||||
webmention.ScannedAt = time.Now()
|
||||
|
||||
html, err := post.user.repo.HttpClient.Get(webmention.Target)
|
||||
if err != nil {
|
||||
// TODO handle error
|
||||
webmention.Supported = false
|
||||
return err
|
||||
}
|
||||
endpoint, err := post.user.repo.Parser.GetWebmentionEndpoint(html)
|
||||
if err != nil {
|
||||
// TODO handle error
|
||||
webmention.Supported = false
|
||||
return err
|
||||
}
|
||||
webmention.Supported = true
|
||||
|
||||
// send webmention
|
||||
payload := url.Values{}
|
||||
payload.Set("source", post.FullUrl())
|
||||
payload.Set("target", webmention.Target)
|
||||
_, err = post.user.repo.HttpClient.Post(endpoint, payload)
|
||||
|
||||
if err != nil {
|
||||
// TODO handle error
|
||||
return err
|
||||
}
|
||||
|
||||
// update webmention status
|
||||
webmention.LastSentAt = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
|
102
post_test.go
102
post_test.go
|
@ -173,7 +173,7 @@ func TestPersistWebmention(t *testing.T) {
|
|||
repo := getTestRepo()
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
webmention := owl.Webmention{
|
||||
webmention := owl.WebmentionIn{
|
||||
Source: "http://example.com/source",
|
||||
}
|
||||
err := post.PersistWebmention(webmention)
|
||||
|
@ -192,8 +192,8 @@ func TestPersistWebmention(t *testing.T) {
|
|||
|
||||
func TestAddWebmentionCreatesFile(t *testing.T) {
|
||||
repo := getTestRepo()
|
||||
repo.Retriever = &MockHttpRetriever{}
|
||||
repo.Parser = &MockMicroformatParser{}
|
||||
repo.HttpClient = &MockHttpRetriever{}
|
||||
repo.Parser = &MockHttpParser{}
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
|
@ -210,8 +210,8 @@ func TestAddWebmentionCreatesFile(t *testing.T) {
|
|||
|
||||
func TestAddWebmentionNotOverwritingFile(t *testing.T) {
|
||||
repo := getTestRepo()
|
||||
repo.Retriever = &MockHttpRetriever{}
|
||||
repo.Parser = &MockMicroformatParser{}
|
||||
repo.HttpClient = &MockHttpRetriever{}
|
||||
repo.Parser = &MockHttpParser{}
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
|
@ -240,8 +240,8 @@ func TestAddWebmentionNotOverwritingFile(t *testing.T) {
|
|||
|
||||
func TestAddWebmentionAddsParsedTitle(t *testing.T) {
|
||||
repo := getTestRepo()
|
||||
repo.Retriever = &MockHttpRetriever{}
|
||||
repo.Parser = &MockMicroformatParser{}
|
||||
repo.HttpClient = &MockHttpRetriever{}
|
||||
repo.Parser = &MockHttpParser{}
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
|
@ -265,25 +265,25 @@ func TestApprovedWebmentions(t *testing.T) {
|
|||
repo := getTestRepo()
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
webmention := owl.Webmention{
|
||||
webmention := owl.WebmentionIn{
|
||||
Source: "http://example.com/source",
|
||||
ApprovalStatus: "approved",
|
||||
RetrievedAt: time.Now(),
|
||||
}
|
||||
post.PersistWebmention(webmention)
|
||||
webmention = owl.Webmention{
|
||||
webmention = owl.WebmentionIn{
|
||||
Source: "http://example.com/source2",
|
||||
ApprovalStatus: "",
|
||||
RetrievedAt: time.Now().Add(time.Hour * -1),
|
||||
}
|
||||
post.PersistWebmention(webmention)
|
||||
webmention = owl.Webmention{
|
||||
webmention = owl.WebmentionIn{
|
||||
Source: "http://example.com/source3",
|
||||
ApprovalStatus: "approved",
|
||||
RetrievedAt: time.Now().Add(time.Hour * -2),
|
||||
}
|
||||
post.PersistWebmention(webmention)
|
||||
webmention = owl.Webmention{
|
||||
webmention = owl.WebmentionIn{
|
||||
Source: "http://example.com/source4",
|
||||
ApprovalStatus: "rejected",
|
||||
RetrievedAt: time.Now().Add(time.Hour * -3),
|
||||
|
@ -303,3 +303,83 @@ func TestApprovedWebmentions(t *testing.T) {
|
|||
}
|
||||
|
||||
}
|
||||
|
||||
func TestScanningForLinks(t *testing.T) {
|
||||
repo := getTestRepo()
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
content := "---\n"
|
||||
content += "title: test\n"
|
||||
content += "date: Wed, 17 Aug 2022 10:50:02 +0000\n"
|
||||
content += "---\n"
|
||||
content += "\n"
|
||||
content += "[Hello](https://example.com/hello)\n"
|
||||
os.WriteFile(post.ContentFile(), []byte(content), 0644)
|
||||
|
||||
post.ScanForLinks()
|
||||
webmentions := post.OutgoingWebmentions()
|
||||
if len(webmentions) != 1 {
|
||||
t.Errorf("Expected 1 webmention, got %d", len(webmentions))
|
||||
}
|
||||
if webmentions[0].Target != "https://example.com/hello" {
|
||||
t.Errorf("Expected target: %s, got %s", "https://example.com/hello", webmentions[0].Target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanningForLinksDoesNotAddDuplicates(t *testing.T) {
|
||||
repo := getTestRepo()
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
content := "---\n"
|
||||
content += "title: test\n"
|
||||
content += "date: Wed, 17 Aug 2022 10:50:02 +0000\n"
|
||||
content += "---\n"
|
||||
content += "\n"
|
||||
content += "[Hello](https://example.com/hello)\n"
|
||||
content += "[Hello](https://example.com/hello)\n"
|
||||
os.WriteFile(post.ContentFile(), []byte(content), 0644)
|
||||
|
||||
post.ScanForLinks()
|
||||
post.ScanForLinks()
|
||||
post.ScanForLinks()
|
||||
webmentions := post.OutgoingWebmentions()
|
||||
if len(webmentions) != 1 {
|
||||
t.Errorf("Expected 1 webmention, got %d", len(webmentions))
|
||||
}
|
||||
if webmentions[0].Target != "https://example.com/hello" {
|
||||
t.Errorf("Expected target: %s, got %s", "https://example.com/hello", webmentions[0].Target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanSendWebmention(t *testing.T) {
|
||||
repo := getTestRepo()
|
||||
repo.HttpClient = &MockHttpRetriever{}
|
||||
repo.Parser = &MockHttpParser{}
|
||||
user, _ := repo.CreateUser("testuser")
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
|
||||
webmention := owl.WebmentionOut{
|
||||
Target: "http://example.com",
|
||||
}
|
||||
|
||||
err := post.SendWebmention(webmention)
|
||||
if err != nil {
|
||||
t.Errorf("Error sending webmention: %v", err)
|
||||
}
|
||||
|
||||
webmentions := post.OutgoingWebmentions()
|
||||
|
||||
if len(webmentions) != 1 {
|
||||
t.Errorf("Expected 1 webmention, got %d", len(webmentions))
|
||||
}
|
||||
|
||||
if webmentions[0].Target != "http://example.com" {
|
||||
t.Errorf("Expected target: %s, got %s", "http://example.com", webmentions[0].Target)
|
||||
}
|
||||
|
||||
if webmentions[0].LastSentAt.IsZero() {
|
||||
t.Errorf("Expected LastSentAt to be set")
|
||||
}
|
||||
}
|
||||
|
|
|
@ -68,13 +68,18 @@ func renderIntoBaseTemplate(user User, data PageContent) (string, error) {
|
|||
return html.String(), nil
|
||||
}
|
||||
|
||||
func RenderPost(post *Post) (string, error) {
|
||||
func renderPostContent(post *Post) (string, error) {
|
||||
buf := post.RenderedContent()
|
||||
postHtml, err := renderEmbedTemplate("embed/post.html", PostRenderData{
|
||||
Title: post.Title(),
|
||||
Post: post,
|
||||
Content: template.HTML(buf.String()),
|
||||
})
|
||||
return postHtml, err
|
||||
}
|
||||
|
||||
func RenderPost(post *Post) (string, error) {
|
||||
postHtml, err := renderPostContent(post)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
|
|
@ -162,14 +162,14 @@ func TestRenderPostIncludesRelToWebMention(t *testing.T) {
|
|||
func TestRenderPostAddsLinksToApprovedWebmention(t *testing.T) {
|
||||
user := getTestUser()
|
||||
post, _ := user.CreateNewPost("testpost")
|
||||
webmention := owl.Webmention{
|
||||
webmention := owl.WebmentionIn{
|
||||
Source: "http://example.com/source3",
|
||||
Title: "Test Title",
|
||||
ApprovalStatus: "approved",
|
||||
RetrievedAt: time.Now().Add(time.Hour * -2),
|
||||
}
|
||||
post.PersistWebmention(webmention)
|
||||
webmention = owl.Webmention{
|
||||
webmention = owl.WebmentionIn{
|
||||
Source: "http://example.com/source4",
|
||||
ApprovalStatus: "rejected",
|
||||
RetrievedAt: time.Now().Add(time.Hour * -3),
|
||||
|
|
|
@ -20,8 +20,8 @@ type Repository struct {
|
|||
single_user_mode bool
|
||||
active_user string
|
||||
allow_raw_html bool
|
||||
Retriever HttpRetriever
|
||||
Parser MicroformatParser
|
||||
HttpClient HttpClient
|
||||
Parser HtmlParser
|
||||
}
|
||||
|
||||
type RepoConfig struct {
|
||||
|
@ -29,7 +29,7 @@ type RepoConfig struct {
|
|||
}
|
||||
|
||||
func CreateRepository(name string) (Repository, error) {
|
||||
newRepo := Repository{name: name, Parser: OwlMicroformatParser{}, Retriever: OwlHttpRetriever{}}
|
||||
newRepo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}}
|
||||
// check if repository already exists
|
||||
if dirExists(newRepo.Dir()) {
|
||||
return Repository{}, fmt.Errorf("Repository already exists")
|
||||
|
@ -63,7 +63,7 @@ func CreateRepository(name string) (Repository, error) {
|
|||
|
||||
func OpenRepository(name string) (Repository, error) {
|
||||
|
||||
repo := Repository{name: name, Parser: OwlMicroformatParser{}, Retriever: OwlHttpRetriever{}}
|
||||
repo := Repository{name: name, Parser: OwlHtmlParser{}, HttpClient: OwlHttpClient{}}
|
||||
if !dirExists(repo.Dir()) {
|
||||
return Repository{}, fmt.Errorf("Repository does not exist: " + repo.Dir())
|
||||
}
|
||||
|
|
|
@ -4,36 +4,47 @@ import (
|
|||
"bytes"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
type Webmention struct {
|
||||
type WebmentionIn struct {
|
||||
Source string `yaml:"source"`
|
||||
Title string `yaml:"title"`
|
||||
ApprovalStatus string `yaml:"approval_status"`
|
||||
RetrievedAt time.Time `yaml:"retrieved_at"`
|
||||
}
|
||||
|
||||
type HttpRetriever interface {
|
||||
type WebmentionOut struct {
|
||||
Target string `yaml:"target"`
|
||||
Supported bool `yaml:"supported"`
|
||||
ScannedAt time.Time `yaml:"scanned_at"`
|
||||
LastSentAt time.Time `yaml:"last_sent_at"`
|
||||
}
|
||||
|
||||
type HttpClient interface {
|
||||
Get(url string) ([]byte, error)
|
||||
Post(url string, data url.Values) ([]byte, error)
|
||||
}
|
||||
|
||||
type MicroformatParser interface {
|
||||
type HtmlParser interface {
|
||||
ParseHEntry(data []byte) (ParsedHEntry, error)
|
||||
ParseLinks(data []byte) ([]string, error)
|
||||
GetWebmentionEndpoint(data []byte) (string, error)
|
||||
}
|
||||
|
||||
type OwlHttpRetriever struct{}
|
||||
type OwlHttpClient struct{}
|
||||
|
||||
type OwlMicroformatParser struct{}
|
||||
type OwlHtmlParser struct{}
|
||||
|
||||
type ParsedHEntry struct {
|
||||
Title string
|
||||
}
|
||||
|
||||
func (OwlHttpRetriever) Get(url string) ([]byte, error) {
|
||||
func (OwlHttpClient) Get(url string) ([]byte, error) {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
|
@ -44,6 +55,17 @@ func (OwlHttpRetriever) Get(url string) ([]byte, error) {
|
|||
return data, err
|
||||
}
|
||||
|
||||
func (OwlHttpClient) Post(url string, data url.Values) ([]byte, error) {
|
||||
resp, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
var respData []byte
|
||||
_, err = resp.Body.Read(respData)
|
||||
|
||||
return respData, err
|
||||
}
|
||||
|
||||
func collectText(n *html.Node, buf *bytes.Buffer) {
|
||||
if n.Type == html.TextNode {
|
||||
buf.WriteString(n.Data)
|
||||
|
@ -53,7 +75,7 @@ func collectText(n *html.Node, buf *bytes.Buffer) {
|
|||
}
|
||||
}
|
||||
|
||||
func (OwlMicroformatParser) ParseHEntry(data []byte) (ParsedHEntry, error) {
|
||||
func (OwlHtmlParser) ParseHEntry(data []byte) (ParsedHEntry, error) {
|
||||
doc, err := html.Parse(strings.NewReader(string(data)))
|
||||
if err != nil {
|
||||
return ParsedHEntry{}, err
|
||||
|
@ -95,3 +117,59 @@ func (OwlMicroformatParser) ParseHEntry(data []byte) (ParsedHEntry, error) {
|
|||
}
|
||||
return findHFeed(doc)
|
||||
}
|
||||
|
||||
func (OwlHtmlParser) ParseLinks(data []byte) ([]string, error) {
|
||||
doc, err := html.Parse(strings.NewReader(string(data)))
|
||||
if err != nil {
|
||||
return make([]string, 0), err
|
||||
}
|
||||
|
||||
var findLinks func(*html.Node) ([]string, error)
|
||||
findLinks = func(n *html.Node) ([]string, error) {
|
||||
links := make([]string, 0)
|
||||
if n.Type == html.ElementNode && n.Data == "a" {
|
||||
for _, attr := range n.Attr {
|
||||
if attr.Key == "href" {
|
||||
links = append(links, attr.Val)
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
childLinks, _ := findLinks(c)
|
||||
links = append(links, childLinks...)
|
||||
}
|
||||
return links, nil
|
||||
}
|
||||
return findLinks(doc)
|
||||
|
||||
}
|
||||
|
||||
func (OwlHtmlParser) GetWebmentionEndpoint(data []byte) (string, error) {
|
||||
doc, err := html.Parse(strings.NewReader(string(data)))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var findEndpoint func(*html.Node) (string, error)
|
||||
findEndpoint = func(n *html.Node) (string, error) {
|
||||
if n.Type == html.ElementNode && n.Data == "link" {
|
||||
for _, attr := range n.Attr {
|
||||
if attr.Key == "rel" && attr.Val == "webmention" {
|
||||
for _, attr := range n.Attr {
|
||||
if attr.Key == "href" {
|
||||
return attr.Val, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
endpoint, err := findEndpoint(c)
|
||||
if err == nil {
|
||||
return endpoint, nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("no webmention endpoint found")
|
||||
}
|
||||
return findEndpoint(doc)
|
||||
}
|
||||
|
|
|
@ -11,7 +11,7 @@ import (
|
|||
|
||||
func TestParseValidHEntry(t *testing.T) {
|
||||
html := []byte("<div class=\"h-entry\"><div class=\"p-name\">Foo</div></div>")
|
||||
parser := &owl.OwlMicroformatParser{}
|
||||
parser := &owl.OwlHtmlParser{}
|
||||
entry, err := parser.ParseHEntry(html)
|
||||
|
||||
if err != nil {
|
||||
|
@ -24,7 +24,7 @@ func TestParseValidHEntry(t *testing.T) {
|
|||
|
||||
func TestParseValidHEntryWithoutTitle(t *testing.T) {
|
||||
html := []byte("<div class=\"h-entry\"></div><div class=\"p-name\">Foo</div>")
|
||||
parser := &owl.OwlMicroformatParser{}
|
||||
parser := &owl.OwlHtmlParser{}
|
||||
entry, err := parser.ParseHEntry(html)
|
||||
|
||||
if err != nil {
|
||||
|
|
Loading…
Reference in New Issue