From b1c46a86aacdb62130d66dbf0cc151f14f042faf Mon Sep 17 00:00:00 2001 From: Niko Abeler Date: Tue, 8 Aug 2023 20:17:04 +0200 Subject: [PATCH 1/5] layouting webmention --- app/entry_register.go | 51 +------------------------ app/generic_register.go | 57 ++++++++++++++++++++++++++++ app/interaction_register.go | 11 ++++++ app/repository/interfaces.go | 8 ++++ app/webmention_service.go | 18 +++++++++ cmd/owl/main.go | 47 ++++++++++++++--------- domain/model/interaction.go | 53 ++++++++++++++++++++++++++ infra/entry_repository.go | 40 ++++++++++---------- infra/interaction_repository.go | 67 +++++++++++++++++++++++++++++++++ interactions/webmention.go | 25 ++++++++++++ web/app.go | 1 + 11 files changed, 292 insertions(+), 86 deletions(-) create mode 100644 app/generic_register.go create mode 100644 app/interaction_register.go create mode 100644 app/webmention_service.go create mode 100644 domain/model/interaction.go create mode 100644 infra/interaction_repository.go create mode 100644 interactions/webmention.go diff --git a/app/entry_register.go b/app/entry_register.go index 90c4e7e..125ae96 100644 --- a/app/entry_register.go +++ b/app/entry_register.go @@ -1,58 +1,11 @@ package app import ( - "errors" "owl-blogs/domain/model" - "reflect" ) -type EntryTypeRegistry struct { - types map[string]model.Entry -} +type EntryTypeRegistry = TypeRegistry[model.Entry] func NewEntryTypeRegistry() *EntryTypeRegistry { - return &EntryTypeRegistry{types: map[string]model.Entry{}} -} - -func (r *EntryTypeRegistry) entryType(entry model.Entry) string { - return reflect.TypeOf(entry).Elem().Name() -} - -func (r *EntryTypeRegistry) Register(entry model.Entry) error { - t := r.entryType(entry) - if _, ok := r.types[t]; ok { - return errors.New("entry type already registered") - } - r.types[t] = entry - return nil -} - -func (r *EntryTypeRegistry) Types() []model.Entry { - types := []model.Entry{} - for _, t := range r.types { - types = append(types, t) - } - return types -} - -func (r *EntryTypeRegistry) TypeName(entry model.Entry) (string, error) { - t := r.entryType(entry) - if _, ok := r.types[t]; !ok { - return "", errors.New("entry type not registered") - } - return t, nil -} - -func (r *EntryTypeRegistry) Type(name string) (model.Entry, error) { - if _, ok := r.types[name]; !ok { - return nil, errors.New("entry type not registered") - } - - val := reflect.ValueOf(r.types[name]) - if val.Kind() == reflect.Ptr { - val = reflect.Indirect(val) - } - newEntry := reflect.New(val.Type()).Interface().(model.Entry) - - return newEntry, nil + return NewTypeRegistry[model.Entry]() } diff --git a/app/generic_register.go b/app/generic_register.go new file mode 100644 index 0000000..8766c70 --- /dev/null +++ b/app/generic_register.go @@ -0,0 +1,57 @@ +package app + +import ( + "errors" + "reflect" +) + +type TypeRegistry[T any] struct { + types map[string]T +} + +func NewTypeRegistry[T any]() *TypeRegistry[T] { + return &TypeRegistry[T]{types: map[string]T{}} +} + +func (r *TypeRegistry[T]) entryType(entry T) string { + return reflect.TypeOf(entry).Elem().Name() +} + +func (r *TypeRegistry[T]) Register(entry T) error { + t := r.entryType(entry) + if _, ok := r.types[t]; ok { + return errors.New("entry type already registered") + } + r.types[t] = entry + return nil +} + +func (r *TypeRegistry[T]) Types() []T { + types := []T{} + for _, t := range r.types { + types = append(types, t) + } + return types +} + +func (r *TypeRegistry[T]) TypeName(entry T) (string, error) { + t := r.entryType(entry) + if _, ok := r.types[t]; !ok { + return "", errors.New("entry type not registered") + } + return t, nil +} + +func (r *TypeRegistry[T]) Type(name string) (T, error) { + if _, ok := r.types[name]; !ok { + return *new(T), errors.New("entry type not registered") + } + + val := reflect.ValueOf(r.types[name]) + if val.Kind() == reflect.Ptr { + val = reflect.Indirect(val) + } + newEntry := reflect.New(val.Type()).Interface().(T) + + return newEntry, nil +} diff --git a/app/interaction_register.go b/app/interaction_register.go new file mode 100644 index 0000000..c1c386b --- /dev/null +++ b/app/interaction_register.go @@ -0,0 +1,11 @@ +package app + +import ( + "owl-blogs/domain/model" +) + +type InteractionTypeRegistry = TypeRegistry[model.Interaction] + +func NewInteractionTypeRegistry() *InteractionTypeRegistry { + return NewTypeRegistry[model.Interaction]() +} diff --git a/app/repository/interfaces.go b/app/repository/interfaces.go index 7e671a4..262ed0e 100644 --- a/app/repository/interfaces.go +++ b/app/repository/interfaces.go @@ -36,3 +36,11 @@ type ConfigRepository interface { Get(name string, config interface{}) error Update(name string, siteConfig interface{}) error } + +type InteractionRepository interface { + Create(interaction model.Interaction) error + Update(interaction model.Interaction) error + Delete(interaction model.Interaction) error + FindById(id string) (model.Interaction, error) + FindAll(entryId string) ([]model.Interaction, error) +} diff --git a/app/webmention_service.go b/app/webmention_service.go new file mode 100644 index 0000000..c348204 --- /dev/null +++ b/app/webmention_service.go @@ -0,0 +1,18 @@ +package app + +import "owl-blogs/app/repository" + +type WebmentionService struct { + InteractionRepository repository.InteractionRepository + EntryRepository repository.EntryRepository +} + +func NewWebmentionService( + interactionRepository repository.InteractionRepository, + entryRepository repository.EntryRepository, +) *WebmentionService { + return &WebmentionService{ + InteractionRepository: interactionRepository, + EntryRepository: entryRepository, + } +} diff --git a/cmd/owl/main.go b/cmd/owl/main.go index ae74c6e..4a967a3 100644 --- a/cmd/owl/main.go +++ b/cmd/owl/main.go @@ -6,6 +6,7 @@ import ( "owl-blogs/app" entrytypes "owl-blogs/entry_types" "owl-blogs/infra" + "owl-blogs/interactions" "owl-blogs/web" "github.com/spf13/cobra" @@ -26,29 +27,41 @@ func Execute() { } func App(db infra.Database) *web.WebApp { - registry := app.NewEntryTypeRegistry() - registry.Register(&entrytypes.Image{}) - registry.Register(&entrytypes.Article{}) - registry.Register(&entrytypes.Page{}) - registry.Register(&entrytypes.Recipe{}) - registry.Register(&entrytypes.Note{}) - registry.Register(&entrytypes.Bookmark{}) - registry.Register(&entrytypes.Reply{}) + // Register Types + entryRegister := app.NewEntryTypeRegistry() + entryRegister.Register(&entrytypes.Image{}) + entryRegister.Register(&entrytypes.Article{}) + entryRegister.Register(&entrytypes.Page{}) + entryRegister.Register(&entrytypes.Recipe{}) + entryRegister.Register(&entrytypes.Note{}) + entryRegister.Register(&entrytypes.Bookmark{}) + entryRegister.Register(&entrytypes.Reply{}) - entryRepo := infra.NewEntryRepository(db, registry) - binRepo := infra.NewBinaryFileRepo(db) - authorRepo := infra.NewDefaultAuthorRepo(db) - siteConfigRepo := infra.NewConfigRepo(db) - - entryService := app.NewEntryService(entryRepo) - binaryService := app.NewBinaryFileService(binRepo) - authorService := app.NewAuthorService(authorRepo, siteConfigRepo) + interactionRegister := app.NewInteractionTypeRegistry() + interactionRegister.Register(&interactions.Webmention{}) configRegister := app.NewConfigRegister() + // Create Repositories + entryRepo := infra.NewEntryRepository(db, entryRegister) + binRepo := infra.NewBinaryFileRepo(db) + authorRepo := infra.NewDefaultAuthorRepo(db) + siteConfigRepo := infra.NewConfigRepo(db) + interactionRepo := infra.NewInteractionRepo(db, interactionRegister) + + // Create Services + entryService := app.NewEntryService(entryRepo) + binaryService := app.NewBinaryFileService(binRepo) + authorService := app.NewAuthorService(authorRepo, siteConfigRepo) + webmentionService := app.NewWebmentionService( + interactionRepo, entryRepo, + ) + + // Create WebApp return web.NewWebApp( - entryService, registry, binaryService, + entryService, entryRegister, binaryService, authorService, siteConfigRepo, configRegister, + webmentionService, ) } diff --git a/domain/model/interaction.go b/domain/model/interaction.go new file mode 100644 index 0000000..5f88215 --- /dev/null +++ b/domain/model/interaction.go @@ -0,0 +1,53 @@ +package model + +import "time" + +type InteractionContent string + +// Interaction is a generic interface for all interactions with entries +// These interactions can be: +// - Webmention, Pingback, Trackback +// - Likes, Comments on third party sites +// - Comments on the site itself +type Interaction interface { + ID() string + EntryID() string + Content() InteractionContent + CreatedAt() time.Time + MetaData() interface{} + + SetID(id string) + SetEntryID(entryID string) + SetCreatedAt(createdAt time.Time) + SetMetaData(metaData interface{}) +} + +type InteractionBase struct { + id string + entryID string + createdAt time.Time +} + +func (i *InteractionBase) ID() string { + return i.id +} + +func (i *InteractionBase) EntryID() string { + return i.entryID +} + +func (i *InteractionBase) CreatedAt() time.Time { + return i.createdAt +} + +func (i *InteractionBase) SetID(id string) { + i.id = id +} + +func (i *InteractionBase) SetEntryID(entryID string) { + i.entryID = entryID +} + +func (i *InteractionBase) SetCreatedAt(createdAt time.Time) { + i.createdAt = createdAt +} diff --git a/infra/entry_repository.go b/infra/entry_repository.go index 1316757..fc2580c 100644 --- a/infra/entry_repository.go +++ b/infra/entry_repository.go @@ -28,6 +28,26 @@ type DefaultEntryRepo struct { db *sqlx.DB } +func NewEntryRepository(db Database, register *app.EntryTypeRegistry) repository.EntryRepository { + sqlxdb := db.Get() + + // Create tables if not exists + sqlxdb.MustExec(` + CREATE TABLE IF NOT EXISTS entries ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + published_at DATETIME, + author_id TEXT NOT NULL, + meta_data TEXT NOT NULL + ); + `) + + return &DefaultEntryRepo{ + db: sqlxdb, + typeRegistry: register, + } +} + // Create implements repository.EntryRepository. func (r *DefaultEntryRepo) Create(entry model.Entry) error { t, err := r.typeRegistry.TypeName(entry) @@ -123,26 +143,6 @@ func (r *DefaultEntryRepo) Update(entry model.Entry) error { return err } -func NewEntryRepository(db Database, register *app.EntryTypeRegistry) repository.EntryRepository { - sqlxdb := db.Get() - - // Create tables if not exists - sqlxdb.MustExec(` - CREATE TABLE IF NOT EXISTS entries ( - id TEXT PRIMARY KEY, - type TEXT NOT NULL, - published_at DATETIME, - author_id TEXT NOT NULL, - meta_data TEXT NOT NULL - ); - `) - - return &DefaultEntryRepo{ - db: sqlxdb, - typeRegistry: register, - } -} - func (r *DefaultEntryRepo) sqlEntryToEntry(entry sqlEntry) (model.Entry, error) { e, err := r.typeRegistry.Type(entry.Type) if err != nil { diff --git a/infra/interaction_repository.go b/infra/interaction_repository.go new file mode 100644 index 0000000..68f4144 --- /dev/null +++ b/infra/interaction_repository.go @@ -0,0 +1,67 @@ +package infra + +import ( + "owl-blogs/app" + "owl-blogs/app/repository" + "owl-blogs/domain/model" + + "github.com/jmoiron/sqlx" +) + +type sqlInteraction struct { + Id string `db:"id"` + Type string `db:"type"` + EntryId string `db:"entry_id"` + CreatedAt string `db:"created_at"` + MetaData string `db:"meta_data"` +} + +type DefaultInteractionRepo struct { + typeRegistry *app.InteractionTypeRegistry + db *sqlx.DB +} + +func NewInteractionRepo(db Database, register *app.InteractionTypeRegistry) repository.InteractionRepository { + sqlxdb := db.Get() + + // Create tables if not exists + sqlxdb.MustExec(` + CREATE TABLE IF NOT EXISTS interactions ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + entry_id TEXT NOT NULL, + created_at DATETIME NOT NULL, + meta_data TEXT NOT NULL + ); + `) + + return &DefaultInteractionRepo{ + db: sqlxdb, + typeRegistry: register, + } +} + +// Create implements repository.InteractionRepository. +func (*DefaultInteractionRepo) Create(interaction model.Interaction) error { + panic("unimplemented") +} + +// Delete implements repository.InteractionRepository. +func (*DefaultInteractionRepo) Delete(interaction model.Interaction) error { + panic("unimplemented") +} + +// FindAll implements repository.InteractionRepository. +func (*DefaultInteractionRepo) FindAll(entryId string) ([]model.Interaction, error) { + panic("unimplemented") +} + +// FindById implements repository.InteractionRepository. +func (*DefaultInteractionRepo) FindById(id string) (model.Interaction, error) { + panic("unimplemented") +} + +// Update implements repository.InteractionRepository. +func (*DefaultInteractionRepo) Update(interaction model.Interaction) error { + panic("unimplemented") +} diff --git a/interactions/webmention.go b/interactions/webmention.go new file mode 100644 index 0000000..24816dc --- /dev/null +++ b/interactions/webmention.go @@ -0,0 +1,25 @@ +package interactions + +import "owl-blogs/domain/model" + +type Webmention struct { + model.InteractionBase + meta WebmentionInteractionMetaData +} + +type WebmentionInteractionMetaData struct { + Source string + Target string +} + +func (i *Webmention) Content() model.InteractionContent { + return model.InteractionContent(i.meta.Source) +} + +func (i *Webmention) MetaData() interface{} { + return &i.meta +} + +func (i *Webmention) SetMetaData(metaData interface{}) { + i.meta = *metaData.(*WebmentionInteractionMetaData) +} diff --git a/web/app.go b/web/app.go index debcfb5..5de6b92 100644 --- a/web/app.go +++ b/web/app.go @@ -34,6 +34,7 @@ func NewWebApp( authorService *app.AuthorService, configRepo repository.ConfigRepository, configRegister *app.ConfigRegister, + webmentionService *app.WebmentionService, ) *WebApp { app := fiber.New() app.Use(middleware.NewUserMiddleware(authorService).Handle) From 6ab9af2d53006d1141aec869889716cfa1ea2595 Mon Sep 17 00:00:00 2001 From: Niko Abeler Date: Tue, 8 Aug 2023 21:32:24 +0200 Subject: [PATCH 2/5] WIP rebuilding incoming webmention from v1 code --- app/owlhttp/interface.go | 13 ++++ app/utils.go | 10 +++ app/webmention_service.go | 150 ++++++++++++++++++++++++++++++++++++- cmd/owl/main.go | 5 +- go.mod | 5 +- go.sum | 6 ++ infra/http.go | 5 ++ interactions/webmention.go | 1 + web/app.go | 3 + web/webmention_handler.go | 55 ++++++++++++++ 10 files changed, 249 insertions(+), 4 deletions(-) create mode 100644 app/owlhttp/interface.go create mode 100644 infra/http.go create mode 100644 web/webmention_handler.go diff --git a/app/owlhttp/interface.go b/app/owlhttp/interface.go new file mode 100644 index 0000000..5997b2e --- /dev/null +++ b/app/owlhttp/interface.go @@ -0,0 +1,13 @@ +package owlhttp + +import ( + "io" + "net/http" + "net/url" +) + +type HttpClient interface { + Get(url string) (resp *http.Response, err error) + Post(url, contentType string, body io.Reader) (resp *http.Response, err error) + PostForm(url string, data url.Values) (resp *http.Response, err error) +} diff --git a/app/utils.go b/app/utils.go index 2797526..b8c3d4c 100644 --- a/app/utils.go +++ b/app/utils.go @@ -2,6 +2,7 @@ package app import ( "math/rand" + "strings" ) var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") @@ -13,3 +14,12 @@ func RandStringRunes(n int) string { } return string(b) } + +func UrlToEntryId(url string) string { + parts := strings.Split(url, "/") + if parts[len(parts)-1] == "" { + return parts[len(parts)-2] + } else { + return parts[len(parts)-1] + } +} diff --git a/app/webmention_service.go b/app/webmention_service.go index c348204..41986b5 100644 --- a/app/webmention_service.go +++ b/app/webmention_service.go @@ -1,18 +1,166 @@ package app -import "owl-blogs/app/repository" +import ( + "bytes" + "errors" + "io" + "net/http" + "owl-blogs/app/owlhttp" + "owl-blogs/app/repository" + "owl-blogs/interactions" + "strings" + "time" + + "golang.org/x/net/html" +) type WebmentionService struct { InteractionRepository repository.InteractionRepository EntryRepository repository.EntryRepository + Http owlhttp.HttpClient +} + +type ParsedHEntry struct { + Title string } func NewWebmentionService( interactionRepository repository.InteractionRepository, entryRepository repository.EntryRepository, + http owlhttp.HttpClient, ) *WebmentionService { return &WebmentionService{ InteractionRepository: interactionRepository, EntryRepository: entryRepository, + Http: http, + } +} + +func readResponseBody(resp *http.Response) (string, error) { + defer resp.Body.Close() + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(bodyBytes), nil +} + +func collectText(n *html.Node, buf *bytes.Buffer) { + + if n.Type == html.TextNode { + buf.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + collectText(c, buf) + } +} + +func (WebmentionService) ParseHEntry(resp *http.Response) (ParsedHEntry, error) { + htmlStr, err := readResponseBody(resp) + if err != nil { + return ParsedHEntry{}, err + } + doc, err := html.Parse(strings.NewReader(htmlStr)) + if err != nil { + return ParsedHEntry{}, err + } + + var interpretHFeed func(*html.Node, *ParsedHEntry, bool) (ParsedHEntry, error) + interpretHFeed = func(n *html.Node, curr *ParsedHEntry, parent bool) (ParsedHEntry, error) { + attrs := n.Attr + for _, attr := range attrs { + if attr.Key == "class" && strings.Contains(attr.Val, "p-name") { + buf := &bytes.Buffer{} + collectText(n, buf) + curr.Title = buf.String() + return *curr, nil + } + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + interpretHFeed(c, curr, false) + } + return *curr, nil + } + + var findHFeed func(*html.Node) (ParsedHEntry, error) + findHFeed = func(n *html.Node) (ParsedHEntry, error) { + attrs := n.Attr + for _, attr := range attrs { + if attr.Key == "class" && strings.Contains(attr.Val, "h-entry") { + return interpretHFeed(n, &ParsedHEntry{}, true) + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + entry, err := findHFeed(c) + if err == nil { + return entry, nil + } + } + return ParsedHEntry{}, errors.New("no h-entry found") + } + return findHFeed(doc) +} + +func (s *WebmentionService) GetExistingWebmention(entryId string, source string, target string) (*interactions.Webmention, error) { + inters, err := s.InteractionRepository.FindAll(entryId) + if err != nil { + return nil, err + } + for _, interaction := range inters { + if webm, ok := interaction.(*interactions.Webmention); ok { + m := webm.MetaData().(interactions.WebmentionInteractionMetaData) + if m.Source == source && m.Target == target { + return webm, nil + } + } + } + return nil, nil +} + +func (s *WebmentionService) ProcessWebmention(source string, target string) error { + resp, err := s.Http.Get(source) + if err != nil { + return err + } + + hEntry, err := s.ParseHEntry(resp) + if err != nil { + return err + } + + entryId := UrlToEntryId(target) + _, err = s.EntryRepository.FindById(entryId) + if err != nil { + return err + } + + webmention, err := s.GetExistingWebmention(entryId, source, target) + if err != nil { + return err + } + if webmention != nil { + data := interactions.WebmentionInteractionMetaData{ + Source: source, + Target: target, + Title: hEntry.Title, + } + webmention.SetMetaData(data) + webmention.SetEntryID(entryId) + webmention.SetCreatedAt(time.Now()) + err = s.InteractionRepository.Update(webmention) + return err + } else { + webmention = &interactions.Webmention{} + data := interactions.WebmentionInteractionMetaData{ + Source: source, + Target: target, + Title: hEntry.Title, + } + webmention.SetMetaData(data) + webmention.SetEntryID(entryId) + webmention.SetCreatedAt(time.Now()) + err = s.InteractionRepository.Create(webmention) + return err } } diff --git a/cmd/owl/main.go b/cmd/owl/main.go index 4a967a3..a86661e 100644 --- a/cmd/owl/main.go +++ b/cmd/owl/main.go @@ -49,12 +49,15 @@ func App(db infra.Database) *web.WebApp { siteConfigRepo := infra.NewConfigRepo(db) interactionRepo := infra.NewInteractionRepo(db, interactionRegister) + // Create External Services + httpClient := &infra.OwlHttpClient{} + // Create Services entryService := app.NewEntryService(entryRepo) binaryService := app.NewBinaryFileService(binRepo) authorService := app.NewAuthorService(authorRepo, siteConfigRepo) webmentionService := app.NewWebmentionService( - interactionRepo, entryRepo, + interactionRepo, entryRepo, httpClient, ) // Create WebApp diff --git a/go.mod b/go.mod index 1af7299..0c45be0 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 github.com/yuin/goldmark v1.5.4 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -37,6 +37,7 @@ require ( github.com/valyala/fasthttp v1.47.0 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/valyala/tcplisten v1.0.0 // indirect - golang.org/x/sys v0.10.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a5a4c5e..01abbac 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -85,6 +87,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -101,6 +105,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= diff --git a/infra/http.go b/infra/http.go new file mode 100644 index 0000000..080cf6d --- /dev/null +++ b/infra/http.go @@ -0,0 +1,5 @@ +package infra + +import "net/http" + +type OwlHttpClient = http.Client diff --git a/interactions/webmention.go b/interactions/webmention.go index 24816dc..22247a9 100644 --- a/interactions/webmention.go +++ b/interactions/webmention.go @@ -10,6 +10,7 @@ type Webmention struct { type WebmentionInteractionMetaData struct { Source string Target string + Title string } func (i *Webmention) Content() model.InteractionContent { diff --git a/web/app.go b/web/app.go index 5de6b92..e3ca712 100644 --- a/web/app.go +++ b/web/app.go @@ -46,6 +46,7 @@ func NewWebApp( rssHandler := NewRSSHandler(entryService, configRepo) loginHandler := NewLoginHandler(authorService, configRepo) editorHandler := NewEditorHandler(entryService, typeRegistry, binService, configRepo) + webmentionHandler := NewWebmentionHandler(webmentionService, configRepo) // Login app.Get("/auth/login", loginHandler.HandleGet) @@ -112,6 +113,8 @@ func NewWebApp( app.Get("/index.xml", rssHandler.Handle) // Posts app.Get("/posts/:post/", entryHandler.Handle) + // Webmention + app.Post("/webmention/", webmentionHandler.Handle) // robots.txt app.Get("/robots.txt", func(c *fiber.Ctx) error { siteConfig := model.SiteConfig{} diff --git a/web/webmention_handler.go b/web/webmention_handler.go new file mode 100644 index 0000000..79e9616 --- /dev/null +++ b/web/webmention_handler.go @@ -0,0 +1,55 @@ +package web + +import ( + "owl-blogs/app" + "owl-blogs/app/repository" + + "github.com/gofiber/fiber/v2" +) + +type WebmentionHandler struct { + configRepo repository.ConfigRepository + webmentionService *app.WebmentionService +} + +func NewWebmentionHandler( + webmentionService *app.WebmentionService, + configRepo repository.ConfigRepository, +) *WebmentionHandler { + return &WebmentionHandler{ + webmentionService: webmentionService, + configRepo: configRepo, + } +} + +func (h *WebmentionHandler) Handle(c *fiber.Ctx) error { + target := c.FormValue("target") + source := c.FormValue("source") + + if target == "" { + return c.Status(400).SendString("target is required") + } + if source == "" { + return c.Status(400).SendString("source is required") + } + + if len(target) < 7 || (target[:7] != "http://" && target[:8] != "https://") { + return c.Status(400).SendString("target must be a valid URL") + } + + if len(source) < 7 || (source[:7] != "http://" && source[:8] != "https://") { + return c.Status(400).SendString("source must be a valid URL") + } + + if source == target { + return c.Status(400).SendString("source and target must be different") + } + + err := h.webmentionService.ProcessWebmention(source, target) + if err != nil { + return err + } + + return c.SendString("ok") + +} From df5215d9432ec502edd5e1689c08446b1c672453 Mon Sep 17 00:00:00 2001 From: Niko Abeler Date: Wed, 9 Aug 2023 20:10:51 +0200 Subject: [PATCH 3/5] tests for incoming webmention --- app/webmention_service.go | 12 +- app/webmention_test.go | 222 ++++++++++++++++++++++++++++++++ infra/interaction_repository.go | 80 ++++++++++-- interactions/webmention.go | 6 +- 4 files changed, 304 insertions(+), 16 deletions(-) create mode 100644 app/webmention_test.go diff --git a/app/webmention_service.go b/app/webmention_service.go index 41986b5..1732d4f 100644 --- a/app/webmention_service.go +++ b/app/webmention_service.go @@ -109,7 +109,7 @@ func (s *WebmentionService) GetExistingWebmention(entryId string, source string, } for _, interaction := range inters { if webm, ok := interaction.(*interactions.Webmention); ok { - m := webm.MetaData().(interactions.WebmentionInteractionMetaData) + m := webm.MetaData().(interactions.WebmentionMetaData) if m.Source == source && m.Target == target { return webm, nil } @@ -130,6 +130,10 @@ func (s *WebmentionService) ProcessWebmention(source string, target string) erro } entryId := UrlToEntryId(target) + println(entryId) + println(entryId) + println(entryId) + println(entryId) _, err = s.EntryRepository.FindById(entryId) if err != nil { return err @@ -140,7 +144,7 @@ func (s *WebmentionService) ProcessWebmention(source string, target string) erro return err } if webmention != nil { - data := interactions.WebmentionInteractionMetaData{ + data := interactions.WebmentionMetaData{ Source: source, Target: target, Title: hEntry.Title, @@ -152,12 +156,12 @@ func (s *WebmentionService) ProcessWebmention(source string, target string) erro return err } else { webmention = &interactions.Webmention{} - data := interactions.WebmentionInteractionMetaData{ + data := interactions.WebmentionMetaData{ Source: source, Target: target, Title: hEntry.Title, } - webmention.SetMetaData(data) + webmention.SetMetaData(&data) webmention.SetEntryID(entryId) webmention.SetCreatedAt(time.Now()) err = s.InteractionRepository.Create(webmention) diff --git a/app/webmention_test.go b/app/webmention_test.go new file mode 100644 index 0000000..29d85bb --- /dev/null +++ b/app/webmention_test.go @@ -0,0 +1,222 @@ +package app_test + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/url" + "owl-blogs/app" + "owl-blogs/infra" + "owl-blogs/interactions" + "owl-blogs/test" + "testing" + + "github.com/stretchr/testify/require" +) + +// func constructResponse(html []byte) *http.Response { +// url, _ := url.Parse("http://example.com/foo/bar") +// return &http.Response{ +// Request: &http.Request{ +// URL: url, +// }, +// Body: io.NopCloser(bytes.NewReader([]byte(html))), +// } +// } + +type MockHttpClient struct { + PageContent string +} + +// Post implements owlhttp.HttpClient. +func (MockHttpClient) Post(url string, contentType string, body io.Reader) (resp *http.Response, err error) { + panic("unimplemented") +} + +// PostForm implements owlhttp.HttpClient. +func (MockHttpClient) PostForm(url string, data url.Values) (resp *http.Response, err error) { + panic("unimplemented") +} + +func (c *MockHttpClient) Get(url string) (*http.Response, error) { + return &http.Response{ + Body: io.NopCloser(bytes.NewReader([]byte(c.PageContent))), + }, nil +} + +func getWebmentionService() *app.WebmentionService { + db := test.NewMockDb() + entryRegister := app.NewEntryTypeRegistry() + entryRegister.Register(&test.MockEntry{}) + entryRepo := infra.NewEntryRepository(db, entryRegister) + + interactionRegister := app.NewInteractionTypeRegistry() + interactionRegister.Register(&interactions.Webmention{}) + + interactionRepo := infra.NewInteractionRepo(db, interactionRegister) + + http := infra.OwlHttpClient{} + return app.NewWebmentionService( + interactionRepo, entryRepo, &http, + ) +} + +// +// https://www.w3.org/TR/webmention/#h-webmention-verification +// + +func TestParseValidHEntry(t *testing.T) { + service := getWebmentionService() + html := []byte("
Foo
") + entry, err := service.ParseHEntry(&http.Response{Body: io.NopCloser(bytes.NewReader(html))}) + + require.NoError(t, err) + require.Equal(t, entry.Title, "Foo") +} + +func TestParseValidHEntryWithoutTitle(t *testing.T) { + service := getWebmentionService() + html := []byte("
Foo
") + entry, err := service.ParseHEntry(&http.Response{Body: io.NopCloser(bytes.NewReader(html))}) + + require.NoError(t, err) + require.Equal(t, entry.Title, "") +} + +func TestCreateNewWebmention(t *testing.T) { + service := getWebmentionService() + service.Http = &MockHttpClient{ + PageContent: "
Foo
", + } + entry := test.MockEntry{} + service.EntryRepository.Create(&entry) + + err := service.ProcessWebmention( + "http://example.com/foo", + fmt.Sprintf("https.//example.com/posts/%s/", entry.ID()), + ) + require.NoError(t, err) + + inters, err := service.InteractionRepository.FindAll(entry.ID()) + require.NoError(t, err) + require.Equal(t, len(inters), 1) + webm := inters[0].(*interactions.Webmention) + meta := webm.MetaData().(*interactions.WebmentionMetaData) + require.Equal(t, meta.Source, "http://example.com/foo") + require.Equal(t, meta.Target, fmt.Sprintf("https.//example.com/posts/%s/", entry.ID())) + require.Equal(t, meta.Title, "Foo") +} + +// func TestGetWebmentionEndpointLink(t *testing.T) { +// service := getWebmentionService() +// html := []byte("") +// endpoint, err := service.GetWebmentionEndpoint(constructResponse(html)) + +// require.NoError(t, err) + +// require.Equal(t, endpoint, "http://example.com/webmention") +// } + +// func TestGetWebmentionEndpointLinkA(t *testing.T) { +// service := getWebmentionService() +// html := []byte("") +// endpoint, err := service.GetWebmentionEndpoint(constructResponse(html)) + +// require.NoError(t, err) +// require.Equal(t, endpoint, "http://example.com/webmention") +// } + +// func TestGetWebmentionEndpointLinkAFakeWebmention(t *testing.T) { +// service := getWebmentionService() +// html := []byte("") +// endpoint, err := service.GetWebmentionEndpoint(constructResponse(html)) + +// require.NoError(t, err) +// require.Equal(t, endpoint, "http://example.com/webmention") +// } + +// func TestGetWebmentionEndpointLinkHeader(t *testing.T) { +// service := getWebmentionService() +// html := []byte("") +// resp := constructResponse(html) +// resp.Header = http.Header{"Link": []string{"; rel=\"webmention\""}} +// endpoint, err := service.GetWebmentionEndpoint(resp) + +// require.NoError(t, err) +// require.Equal(t, endpoint, "http://example.com/webmention") +// } + +// func TestGetWebmentionEndpointLinkHeaderCommas(t *testing.T) { +// service := getWebmentionService() +// html := []byte("") +// resp := constructResponse(html) +// resp.Header = http.Header{ +// "Link": []string{"; rel=\"other\", ; rel=\"webmention\""}, +// } +// endpoint, err := service.GetWebmentionEndpoint(resp) + +// require.NoError(t, err) +// require.Equal(t, endpoint, "https://webmention.rocks/test/19/webmention") +// } + +// func TestGetWebmentionEndpointRelativeLink(t *testing.T) { +// service := getWebmentionService() +// html := []byte("") +// endpoint, err := service.GetWebmentionEndpoint(constructResponse(html)) + +// require.NoError(t, err) +// require.Equal(t, endpoint, "http://example.com/webmention") +// } + +// func TestGetWebmentionEndpointRelativeLinkInHeader(t *testing.T) { +// service := getWebmentionService() +// html := []byte("") +// resp := constructResponse(html) +// resp.Header = http.Header{"Link": []string{"; rel=\"webmention\""}} +// endpoint, err := service.GetWebmentionEndpoint(resp) + +// require.NoError(t, err) +// require.Equal(t, endpoint, "http://example.com/webmention") +// } + +// func TestRealWorldWebmention(t *testing.T) { +// service := getWebmentionService() +// links := []string{ +// "https://webmention.rocks/test/1", +// "https://webmention.rocks/test/2", +// "https://webmention.rocks/test/3", +// "https://webmention.rocks/test/4", +// "https://webmention.rocks/test/5", +// "https://webmention.rocks/test/6", +// "https://webmention.rocks/test/7", +// "https://webmention.rocks/test/8", +// "https://webmention.rocks/test/9", +// // "https://webmention.rocks/test/10", // not supported +// "https://webmention.rocks/test/11", +// "https://webmention.rocks/test/12", +// "https://webmention.rocks/test/13", +// "https://webmention.rocks/test/14", +// "https://webmention.rocks/test/15", +// "https://webmention.rocks/test/16", +// "https://webmention.rocks/test/17", +// "https://webmention.rocks/test/18", +// "https://webmention.rocks/test/19", +// "https://webmention.rocks/test/20", +// "https://webmention.rocks/test/21", +// "https://webmention.rocks/test/22", +// "https://webmention.rocks/test/23/page", +// } + +// for _, link := range links { +// +// client := &owl.OwlHttpClient{} +// html, _ := client.Get(link) +// _, err := service.GetWebmentionEndpoint(html) + +// if err != nil { +// t.Errorf("Unable to find webmention: %v for link %v", err, link) +// } +// } + +// } diff --git a/infra/interaction_repository.go b/infra/interaction_repository.go index 68f4144..c5b4f83 100644 --- a/infra/interaction_repository.go +++ b/infra/interaction_repository.go @@ -1,19 +1,24 @@ package infra import ( + "encoding/json" + "errors" "owl-blogs/app" "owl-blogs/app/repository" "owl-blogs/domain/model" + "reflect" + "time" + "github.com/google/uuid" "github.com/jmoiron/sqlx" ) type sqlInteraction struct { - Id string `db:"id"` - Type string `db:"type"` - EntryId string `db:"entry_id"` - CreatedAt string `db:"created_at"` - MetaData string `db:"meta_data"` + Id string `db:"id"` + Type string `db:"type"` + EntryId string `db:"entry_id"` + CreatedAt time.Time `db:"created_at"` + MetaData *string `db:"meta_data"` } type DefaultInteractionRepo struct { @@ -42,8 +47,34 @@ func NewInteractionRepo(db Database, register *app.InteractionTypeRegistry) repo } // Create implements repository.InteractionRepository. -func (*DefaultInteractionRepo) Create(interaction model.Interaction) error { - panic("unimplemented") +func (repo *DefaultInteractionRepo) Create(interaction model.Interaction) error { + t, err := repo.typeRegistry.TypeName(interaction) + if err != nil { + return errors.New("interaction type not registered") + } + + if interaction.ID() == "" { + interaction.SetID(uuid.New().String()) + } + + var metaDataJson []byte + if interaction.MetaData() != nil { + metaDataJson, _ = json.Marshal(interaction.MetaData()) + } + metaDataStr := string(metaDataJson) + + _, err = repo.db.NamedExec(` + INSERT INTO interactions (id, type, entry_id, created_at, meta_data) + VALUES (:id, :type, :entry_id, :created_at, :meta_data) + `, sqlInteraction{ + Id: interaction.ID(), + Type: t, + EntryId: interaction.EntryID(), + CreatedAt: interaction.CreatedAt(), + MetaData: &metaDataStr, + }) + + return err } // Delete implements repository.InteractionRepository. @@ -52,8 +83,23 @@ func (*DefaultInteractionRepo) Delete(interaction model.Interaction) error { } // FindAll implements repository.InteractionRepository. -func (*DefaultInteractionRepo) FindAll(entryId string) ([]model.Interaction, error) { - panic("unimplemented") +func (repo *DefaultInteractionRepo) FindAll(entryId string) ([]model.Interaction, error) { + data := []sqlInteraction{} + err := repo.db.Select(&data, "SELECT * FROM interactions WHERE entry_id = ?", entryId) + if err != nil { + return nil, err + } + + interactions := []model.Interaction{} + for _, d := range data { + i, err := repo.sqlInteractionToInteraction(d) + if err != nil { + return nil, err + } + interactions = append(interactions, i) + } + + return interactions, nil } // FindById implements repository.InteractionRepository. @@ -65,3 +111,19 @@ func (*DefaultInteractionRepo) FindById(id string) (model.Interaction, error) { func (*DefaultInteractionRepo) Update(interaction model.Interaction) error { panic("unimplemented") } + +func (repo *DefaultInteractionRepo) sqlInteractionToInteraction(interaction sqlInteraction) (model.Interaction, error) { + i, err := repo.typeRegistry.Type(interaction.Type) + if err != nil { + return nil, errors.New("interaction type not registered") + } + metaData := reflect.New(reflect.TypeOf(i.MetaData()).Elem()).Interface() + json.Unmarshal([]byte(*interaction.MetaData), metaData) + i.SetID(interaction.Id) + i.SetEntryID(interaction.EntryId) + i.SetCreatedAt(interaction.CreatedAt) + i.SetMetaData(metaData) + + return i, nil + +} diff --git a/interactions/webmention.go b/interactions/webmention.go index 22247a9..09d8fc0 100644 --- a/interactions/webmention.go +++ b/interactions/webmention.go @@ -4,10 +4,10 @@ import "owl-blogs/domain/model" type Webmention struct { model.InteractionBase - meta WebmentionInteractionMetaData + meta WebmentionMetaData } -type WebmentionInteractionMetaData struct { +type WebmentionMetaData struct { Source string Target string Title string @@ -22,5 +22,5 @@ func (i *Webmention) MetaData() interface{} { } func (i *Webmention) SetMetaData(metaData interface{}) { - i.meta = *metaData.(*WebmentionInteractionMetaData) + i.meta = *metaData.(*WebmentionMetaData) } From 723a6000bf92d5d71764ed547871fc5a5856d5a0 Mon Sep 17 00:00:00 2001 From: Niko Abeler Date: Wed, 9 Aug 2023 20:36:44 +0200 Subject: [PATCH 4/5] first retrieved webmention --- app/webmention_service.go | 4 +-- cmd/owl/main.go | 2 +- infra/interaction_repository.go | 33 +++++++++++++++--- interactions/webmention.go | 12 +++++-- render/templates/base.tmpl | 1 + render/templates/interaction/Webmention.tmpl | 7 ++++ render/templates/views/entry.tmpl | 19 +++++++++++ web/app.go | 3 +- web/entry_handler.go | 35 ++++++++++++-------- web/webmention_handler.go | 3 ++ 10 files changed, 95 insertions(+), 24 deletions(-) create mode 100644 render/templates/interaction/Webmention.tmpl diff --git a/app/webmention_service.go b/app/webmention_service.go index 1732d4f..940ec77 100644 --- a/app/webmention_service.go +++ b/app/webmention_service.go @@ -109,7 +109,7 @@ func (s *WebmentionService) GetExistingWebmention(entryId string, source string, } for _, interaction := range inters { if webm, ok := interaction.(*interactions.Webmention); ok { - m := webm.MetaData().(interactions.WebmentionMetaData) + m := webm.MetaData().(*interactions.WebmentionMetaData) if m.Source == source && m.Target == target { return webm, nil } @@ -149,7 +149,7 @@ func (s *WebmentionService) ProcessWebmention(source string, target string) erro Target: target, Title: hEntry.Title, } - webmention.SetMetaData(data) + webmention.SetMetaData(&data) webmention.SetEntryID(entryId) webmention.SetCreatedAt(time.Now()) err = s.InteractionRepository.Update(webmention) diff --git a/cmd/owl/main.go b/cmd/owl/main.go index a86661e..724c332 100644 --- a/cmd/owl/main.go +++ b/cmd/owl/main.go @@ -64,7 +64,7 @@ func App(db infra.Database) *web.WebApp { return web.NewWebApp( entryService, entryRegister, binaryService, authorService, siteConfigRepo, configRegister, - webmentionService, + webmentionService, interactionRepo, ) } diff --git a/infra/interaction_repository.go b/infra/interaction_repository.go index c5b4f83..e9e88fe 100644 --- a/infra/interaction_repository.go +++ b/infra/interaction_repository.go @@ -103,13 +103,38 @@ func (repo *DefaultInteractionRepo) FindAll(entryId string) ([]model.Interaction } // FindById implements repository.InteractionRepository. -func (*DefaultInteractionRepo) FindById(id string) (model.Interaction, error) { - panic("unimplemented") +func (repo *DefaultInteractionRepo) FindById(id string) (model.Interaction, error) { + data := sqlInteraction{} + err := repo.db.Get(&data, "SELECT * FROM interactions WHERE id = ?", id) + if err != nil { + return nil, err + } + if data.Id == "" { + return nil, errors.New("interaction not found") + } + return repo.sqlInteractionToInteraction(data) } // Update implements repository.InteractionRepository. -func (*DefaultInteractionRepo) Update(interaction model.Interaction) error { - panic("unimplemented") +func (repo *DefaultInteractionRepo) Update(interaction model.Interaction) error { + exInter, _ := repo.FindById(interaction.ID()) + if exInter == nil { + return errors.New("interaction not found") + } + + _, err := repo.typeRegistry.TypeName(interaction) + if err != nil { + return errors.New("interaction type not registered") + } + + var metaDataJson []byte + if interaction.MetaData() != nil { + metaDataJson, _ = json.Marshal(interaction.MetaData()) + } + + _, err = repo.db.Exec("UPDATE interactions SET entry_id = ?, meta_data = ? WHERE id = ?", interaction.EntryID(), metaDataJson, interaction.ID()) + + return err } func (repo *DefaultInteractionRepo) sqlInteractionToInteraction(interaction sqlInteraction) (model.Interaction, error) { diff --git a/interactions/webmention.go b/interactions/webmention.go index 09d8fc0..d60b45f 100644 --- a/interactions/webmention.go +++ b/interactions/webmention.go @@ -1,6 +1,10 @@ package interactions -import "owl-blogs/domain/model" +import ( + "fmt" + "owl-blogs/domain/model" + "owl-blogs/render" +) type Webmention struct { model.InteractionBase @@ -14,7 +18,11 @@ type WebmentionMetaData struct { } func (i *Webmention) Content() model.InteractionContent { - return model.InteractionContent(i.meta.Source) + str, err := render.RenderTemplateToString("interaction/Webmention", i) + if err != nil { + fmt.Println(err) + } + return model.InteractionContent(str) } func (i *Webmention) MetaData() interface{} { diff --git a/render/templates/base.tmpl b/render/templates/base.tmpl index d0b0815..2fac1dd 100644 --- a/render/templates/base.tmpl +++ b/render/templates/base.tmpl @@ -6,6 +6,7 @@ {{template "title" .Data}} - {{ .SiteConfig.Title }} +