From 6ab9af2d53006d1141aec869889716cfa1ea2595 Mon Sep 17 00:00:00 2001 From: Niko Abeler Date: Tue, 8 Aug 2023 21:32:24 +0200 Subject: [PATCH] WIP rebuilding incoming webmention from v1 code --- app/owlhttp/interface.go | 13 ++++ app/utils.go | 10 +++ app/webmention_service.go | 150 ++++++++++++++++++++++++++++++++++++- cmd/owl/main.go | 5 +- go.mod | 5 +- go.sum | 6 ++ infra/http.go | 5 ++ interactions/webmention.go | 1 + web/app.go | 3 + web/webmention_handler.go | 55 ++++++++++++++ 10 files changed, 249 insertions(+), 4 deletions(-) create mode 100644 app/owlhttp/interface.go create mode 100644 infra/http.go create mode 100644 web/webmention_handler.go diff --git a/app/owlhttp/interface.go b/app/owlhttp/interface.go new file mode 100644 index 0000000..5997b2e --- /dev/null +++ b/app/owlhttp/interface.go @@ -0,0 +1,13 @@ +package owlhttp + +import ( + "io" + "net/http" + "net/url" +) + +type HttpClient interface { + Get(url string) (resp *http.Response, err error) + Post(url, contentType string, body io.Reader) (resp *http.Response, err error) + PostForm(url string, data url.Values) (resp *http.Response, err error) +} diff --git a/app/utils.go b/app/utils.go index 2797526..b8c3d4c 100644 --- a/app/utils.go +++ b/app/utils.go @@ -2,6 +2,7 @@ package app import ( "math/rand" + "strings" ) var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") @@ -13,3 +14,12 @@ func RandStringRunes(n int) string { } return string(b) } + +func UrlToEntryId(url string) string { + parts := strings.Split(url, "/") + if parts[len(parts)-1] == "" { + return parts[len(parts)-2] + } else { + return parts[len(parts)-1] + } +} diff --git a/app/webmention_service.go b/app/webmention_service.go index c348204..41986b5 100644 --- a/app/webmention_service.go +++ b/app/webmention_service.go @@ -1,18 +1,166 @@ package app -import "owl-blogs/app/repository" +import ( + "bytes" + "errors" + "io" + "net/http" + "owl-blogs/app/owlhttp" + "owl-blogs/app/repository" + "owl-blogs/interactions" + "strings" + "time" + + "golang.org/x/net/html" +) type WebmentionService struct { InteractionRepository repository.InteractionRepository EntryRepository repository.EntryRepository + Http owlhttp.HttpClient +} + +type ParsedHEntry struct { + Title string } func NewWebmentionService( interactionRepository repository.InteractionRepository, entryRepository repository.EntryRepository, + http owlhttp.HttpClient, ) *WebmentionService { return &WebmentionService{ InteractionRepository: interactionRepository, EntryRepository: entryRepository, + Http: http, + } +} + +func readResponseBody(resp *http.Response) (string, error) { + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(bodyBytes), nil +} + +func collectText(n *html.Node, buf *bytes.Buffer) { + + if n.Type == html.TextNode { + buf.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + collectText(c, buf) + } +} + +func (WebmentionService) ParseHEntry(resp *http.Response) (ParsedHEntry, error) { + htmlStr, err := readResponseBody(resp) + if err != nil { + return ParsedHEntry{}, err + } + doc, err := html.Parse(strings.NewReader(htmlStr)) + if err != nil { + return ParsedHEntry{}, err + } + + var interpretHFeed func(*html.Node, *ParsedHEntry, bool) (ParsedHEntry, error) + interpretHFeed = func(n *html.Node, curr *ParsedHEntry, parent bool) (ParsedHEntry, error) { + attrs := n.Attr + for _, attr := range attrs { + if attr.Key == "class" && strings.Contains(attr.Val, "p-name") { + buf := &bytes.Buffer{} + collectText(n, buf) + curr.Title = buf.String() + return *curr, nil + } + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + interpretHFeed(c, curr, false) + } + return *curr, nil + } + + var findHFeed func(*html.Node) (ParsedHEntry, error) + findHFeed = func(n *html.Node) (ParsedHEntry, error) { + attrs := n.Attr + for _, attr := range attrs { + if attr.Key == "class" && strings.Contains(attr.Val, "h-entry") { + return interpretHFeed(n, &ParsedHEntry{}, true) + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + entry, err := findHFeed(c) + if err == nil { + return entry, nil + } + } + return ParsedHEntry{}, errors.New("no h-entry found") + } + return findHFeed(doc) +} + +func (s *WebmentionService) GetExistingWebmention(entryId string, source string, target string) (*interactions.Webmention, error) { + inters, err := s.InteractionRepository.FindAll(entryId) + if err != nil { + return nil, err + } + for _, interaction := range inters { + if webm, ok := interaction.(*interactions.Webmention); ok { + m := webm.MetaData().(interactions.WebmentionInteractionMetaData) + if m.Source == source && m.Target == target { + return webm, nil + } + } + } + return nil, nil +} + +func (s *WebmentionService) ProcessWebmention(source string, target string) error { + resp, err := s.Http.Get(source) + if err != nil { + return err + } + + hEntry, err := s.ParseHEntry(resp) + if err != nil { + return err + } + + entryId := UrlToEntryId(target) + _, err = s.EntryRepository.FindById(entryId) + if err != nil { + return err + } + + webmention, err := s.GetExistingWebmention(entryId, source, target) + if err != nil { + return err + } + if webmention != nil { + data := interactions.WebmentionInteractionMetaData{ + Source: source, + Target: target, + Title: hEntry.Title, + } + webmention.SetMetaData(data) + webmention.SetEntryID(entryId) + webmention.SetCreatedAt(time.Now()) + err = s.InteractionRepository.Update(webmention) + return err + } else { + webmention = &interactions.Webmention{} + data := interactions.WebmentionInteractionMetaData{ + Source: source, + Target: target, + Title: hEntry.Title, + } + webmention.SetMetaData(data) + webmention.SetEntryID(entryId) + webmention.SetCreatedAt(time.Now()) + err = s.InteractionRepository.Create(webmention) + return err } } diff --git a/cmd/owl/main.go b/cmd/owl/main.go index 4a967a3..a86661e 100644 --- a/cmd/owl/main.go +++ b/cmd/owl/main.go @@ -49,12 +49,15 @@ func App(db infra.Database) *web.WebApp { siteConfigRepo := infra.NewConfigRepo(db) interactionRepo := infra.NewInteractionRepo(db, interactionRegister) + // Create External Services + httpClient := &infra.OwlHttpClient{} + // Create Services entryService := app.NewEntryService(entryRepo) binaryService := app.NewBinaryFileService(binRepo) authorService := app.NewAuthorService(authorRepo, siteConfigRepo) webmentionService := app.NewWebmentionService( - interactionRepo, entryRepo, + interactionRepo, entryRepo, httpClient, ) // Create WebApp diff --git a/go.mod b/go.mod index 1af7299..0c45be0 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 github.com/yuin/goldmark v1.5.4 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -37,6 +37,7 @@ require ( github.com/valyala/fasthttp v1.47.0 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - golang.org/x/sys v0.10.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a5a4c5e..01abbac 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -85,6 +87,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -101,6 +105,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= diff --git a/infra/http.go b/infra/http.go new file mode 100644 index 0000000..080cf6d --- /dev/null +++ b/infra/http.go @@ -0,0 +1,5 @@ +package infra + +import "net/http" + +type OwlHttpClient = http.Client diff --git a/interactions/webmention.go b/interactions/webmention.go index 24816dc..22247a9 100644 --- a/interactions/webmention.go +++ b/interactions/webmention.go @@ -10,6 +10,7 @@ type Webmention struct { type WebmentionInteractionMetaData struct { Source string Target string + Title string } func (i *Webmention) Content() model.InteractionContent { diff --git a/web/app.go b/web/app.go index 5de6b92..e3ca712 100644 --- a/web/app.go +++ b/web/app.go @@ -46,6 +46,7 @@ func NewWebApp( rssHandler := NewRSSHandler(entryService, configRepo) loginHandler := NewLoginHandler(authorService, configRepo) editorHandler := NewEditorHandler(entryService, typeRegistry, binService, configRepo) + webmentionHandler := NewWebmentionHandler(webmentionService, configRepo) // Login app.Get("/auth/login", loginHandler.HandleGet) @@ -112,6 +113,8 @@ func NewWebApp( app.Get("/index.xml", rssHandler.Handle) // Posts app.Get("/posts/:post/", entryHandler.Handle) + // Webmention + app.Post("/webmention/", webmentionHandler.Handle) // robots.txt app.Get("/robots.txt", func(c *fiber.Ctx) error { siteConfig := model.SiteConfig{} diff --git a/web/webmention_handler.go b/web/webmention_handler.go new file mode 100644 index 0000000..79e9616 --- /dev/null +++ b/web/webmention_handler.go @@ -0,0 +1,55 @@ +package web + +import ( + "owl-blogs/app" + "owl-blogs/app/repository" + + "github.com/gofiber/fiber/v2" +) + +type WebmentionHandler struct { + configRepo repository.ConfigRepository + webmentionService *app.WebmentionService +} + +func NewWebmentionHandler( + webmentionService *app.WebmentionService, + configRepo repository.ConfigRepository, +) *WebmentionHandler { + return &WebmentionHandler{ + webmentionService: webmentionService, + configRepo: configRepo, + } +} + +func (h *WebmentionHandler) Handle(c *fiber.Ctx) error { + target := c.FormValue("target") + source := c.FormValue("source") + + if target == "" { + return c.Status(400).SendString("target is required") + } + if source == "" { + return c.Status(400).SendString("source is required") + } + + if len(target) < 7 || (target[:7] != "http://" && target[:8] != "https://") { + return c.Status(400).SendString("target must be a valid URL") + } + + if len(source) < 7 || (source[:7] != "http://" && source[:8] != "https://") { + return c.Status(400).SendString("source must be a valid URL") + } + + if source == target { + return c.Status(400).SendString("source and target must be different") + } + + err := h.webmentionService.ProcessWebmention(source, target) + if err != nil { + return err + } + + return c.SendString("ok") + +}