From 9c30ff7877ef2ff73366b051d099d8eede74b570 Mon Sep 17 00:00:00 2001 From: Niko Abeler Date: Sun, 9 Jul 2023 22:12:06 +0200 Subject: [PATCH] WIP Import + refactor entry creation --- app/entry_service.go | 5 +- app/repository/interfaces.go | 3 +- cmd/owl/import_v1.go | 53 +++++++-- go.mod | 1 + go.sum | 2 + importer/utils.go | 200 +++++++++++++++++++++++++++++++++ infra/entry_repository.go | 14 +-- infra/entry_repository_test.go | 34 ++++-- web/editor_handler.go | 4 +- 9 files changed, 287 insertions(+), 29 deletions(-) create mode 100644 importer/utils.go diff --git a/app/entry_service.go b/app/entry_service.go index 103acab..ba64baf 100644 --- a/app/entry_service.go +++ b/app/entry_service.go @@ -3,7 +3,6 @@ package app import ( "owl-blogs/app/repository" "owl-blogs/domain/model" - "time" ) type EntryService struct { @@ -14,8 +13,8 @@ func NewEntryService(entryRepository repository.EntryRepository) *EntryService { return &EntryService{EntryRepository: entryRepository} } -func (s *EntryService) Create(entry model.Entry, publishedAt *time.Time, metaData model.EntryMetaData) error { - return s.EntryRepository.Create(entry, publishedAt, metaData) +func (s *EntryService) Create(entry model.Entry) error { + return s.EntryRepository.Create(entry) } func (s *EntryService) Update(entry model.Entry) error { diff --git a/app/repository/interfaces.go b/app/repository/interfaces.go index 15341ae..c5c3013 100644 --- a/app/repository/interfaces.go +++ b/app/repository/interfaces.go @@ -2,11 +2,10 @@ package repository import ( "owl-blogs/domain/model" - "time" ) type EntryRepository interface { - Create(entry model.Entry, publishedAt *time.Time, metaData model.EntryMetaData) error + Create(entry model.Entry) error Update(entry model.Entry) error Delete(entry model.Entry) error FindById(id string) (model.Entry, error) diff --git a/cmd/owl/import_v1.go b/cmd/owl/import_v1.go index 143fd04..5a02868 100644 --- a/cmd/owl/import_v1.go +++ b/cmd/owl/import_v1.go @@ -1,13 +1,20 @@ package main import ( + "fmt" + "owl-blogs/domain/model" + "owl-blogs/importer" + "owl-blogs/infra" + "github.com/spf13/cobra" ) +var userPath string + func init() { rootCmd.AddCommand(importCmd) - importCmd.Flags().StringVarP(&user, "path", "p", "", "Path to the user folder") + importCmd.Flags().StringVarP(&userPath, "path", "p", "", "Path to the user folder") importCmd.MarkFlagRequired("path") } @@ -16,13 +23,43 @@ var importCmd = &cobra.Command{ Short: "Import data from v1", Long: `Import data from v1`, Run: func(cmd *cobra.Command, args []string) { - // db := infra.NewSqliteDB(DbPath) - // App(db).ImportV1() + db := infra.NewSqliteDB(DbPath) + app := App(db) - // TODO: Implement this - // For each folder in the user folder - // Map to entry types - // Convert and save - // Import Binary files + posts, err := importer.AllUserPosts(userPath) + if err != nil { + panic(err) + } + + for _, post := range posts { + fmt.Println(post.Meta.Type) + switch post.Meta.Type { + case "article": + article := model.Article{} + article.SetID(post.Id) + article.SetMetaData(model.ArticleMetaData{ + Title: post.Meta.Title, + Content: post.Content, + }) + article.SetPublishedAt(&post.Meta.Date) + app.EntryService.Create(&article) + + case "bookmark": + + case "reply": + + case "photo": + + case "note": + + case "recipe": + + case "page": + + default: + panic("Unknown type") + } + + } }, } diff --git a/go.mod b/go.mod index 35d83e3..54cae65 100644 --- a/go.mod +++ b/go.mod @@ -29,5 +29,6 @@ require ( github.com/valyala/tcplisten v1.0.0 // indirect golang.org/x/crypto v0.11.0 // indirect golang.org/x/sys v0.10.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d8332d0..a5aaeef 100644 --- a/go.sum +++ b/go.sum @@ -111,6 +111,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/importer/utils.go b/importer/utils.go new file mode 100644 index 0000000..15bf9e2 --- /dev/null +++ b/importer/utils.go @@ -0,0 +1,200 @@ +package importer + +import ( + "bytes" + "os" + "path" + "time" + + "gopkg.in/yaml.v2" +) + +type ReplyData struct { + Url string `yaml:"url"` + Text string `yaml:"text"` +} +type BookmarkData struct { + Url string `yaml:"url"` + Text string `yaml:"text"` +} + +type RecipeData struct { + Yield string `yaml:"yield"` + Duration string `yaml:"duration"` + Ingredients []string `yaml:"ingredients"` +} + +type PostMeta struct { + Type string `yaml:"type"` + Title string `yaml:"title"` + Description string `yaml:"description"` + Aliases []string `yaml:"aliases"` + Date time.Time `yaml:"date"` + Draft bool `yaml:"draft"` + Reply ReplyData `yaml:"reply"` + Bookmark BookmarkData `yaml:"bookmark"` + Recipe RecipeData `yaml:"recipe"` + PhotoPath string `yaml:"photo"` +} + +type Post struct { + Id string + Meta PostMeta + Content string +} + +func (pm *PostMeta) UnmarshalYAML(unmarshal func(interface{}) error) error { + type T struct { + Type string `yaml:"type"` + Title string `yaml:"title"` + Description string `yaml:"description"` + Aliases []string `yaml:"aliases"` + Draft bool `yaml:"draft"` + Reply ReplyData `yaml:"reply"` + Bookmark BookmarkData `yaml:"bookmark"` + Recipe RecipeData `yaml:"recipe"` + PhotoPath string `yaml:"photo"` + } + type S struct { + Date string `yaml:"date"` + } + + var t T + var s S + if err := unmarshal(&t); err != nil { + return err + } + if err := unmarshal(&s); err != nil { + return err + } + + pm.Type = t.Type + if pm.Type == "" { + pm.Type = "article" + } + pm.Title = t.Title + pm.Description = t.Description + pm.Aliases = t.Aliases + pm.Draft = t.Draft + pm.Reply = t.Reply + pm.Bookmark = t.Bookmark + pm.Recipe = t.Recipe + pm.PhotoPath = t.PhotoPath + + possibleFormats := []string{ + "2006-01-02", + time.Layout, + time.ANSIC, + time.UnixDate, + time.RubyDate, + time.RFC822, + time.RFC822Z, + time.RFC850, + time.RFC1123, + time.RFC1123Z, + time.RFC3339, + time.RFC3339Nano, + time.Stamp, + time.StampMilli, + time.StampMicro, + time.StampNano, + } + + for _, format := range possibleFormats { + if t, err := time.Parse(format, s.Date); err == nil { + pm.Date = t + break + } + } + + return nil +} + +func LoadContent(data []byte) string { + + // trim yaml block + // TODO this can be done nicer + trimmedData := bytes.TrimSpace(data) + // ensure that data ends with a newline + trimmedData = append(trimmedData, []byte("\n")...) + // check first line is --- + if string(trimmedData[0:4]) == "---\n" { + trimmedData = trimmedData[4:] + // find --- end + end := bytes.Index(trimmedData, []byte("\n---\n")) + if end != -1 { + data = trimmedData[end+5:] + } + } + + return string(data) +} + +func LoadMeta(data []byte) (PostMeta, error) { + + // get yaml metadata block + meta := PostMeta{} + trimmedData := bytes.TrimSpace(data) + // ensure that data ends with a newline + trimmedData = append(trimmedData, []byte("\n")...) + // check first line is --- + if string(trimmedData[0:4]) == "---\n" { + trimmedData = trimmedData[4:] + // find --- end + end := bytes.Index(trimmedData, []byte("---\n")) + if end != -1 { + metaData := trimmedData[:end] + err := yaml.Unmarshal(metaData, &meta) + if err != nil { + return PostMeta{}, err + } + } + } + + return meta, nil +} + +func AllUserPosts(userPath string) ([]Post, error) { + postFiles := listDir(path.Join(userPath, "public")) + posts := make([]Post, 0) + for _, id := range postFiles { + // if is a directory and has index.md, add to posts + if dirExists(path.Join(userPath, "public", id)) { + if fileExists(path.Join(userPath, "public", id, "index.md")) { + postData, err := os.ReadFile(path.Join(userPath, "public", id, "index.md")) + if err != nil { + return nil, err + } + meta, err := LoadMeta(postData) + if err != nil { + return nil, err + } + post := Post{ + Id: id, + Content: LoadContent(postData), + Meta: meta, + } + posts = append(posts, post) + } + } + } + + return posts, nil +} + +func listDir(path string) []string { + dir, _ := os.Open(path) + defer dir.Close() + files, _ := dir.Readdirnames(-1) + return files +} + +func dirExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/infra/entry_repository.go b/infra/entry_repository.go index 611523f..6f9a7fc 100644 --- a/infra/entry_repository.go +++ b/infra/entry_repository.go @@ -28,7 +28,7 @@ type DefaultEntryRepo struct { } // Create implements repository.EntryRepository. -func (r *DefaultEntryRepo) Create(entry model.Entry, publishedAt *time.Time, metaData model.EntryMetaData) error { +func (r *DefaultEntryRepo) Create(entry model.Entry) error { t, err := r.typeRegistry.TypeName(entry) if err != nil { return errors.New("entry type not registered") @@ -36,14 +36,14 @@ func (r *DefaultEntryRepo) Create(entry model.Entry, publishedAt *time.Time, met var metaDataJson []byte if entry.MetaData() != nil { - metaDataJson, _ = json.Marshal(metaData) + metaDataJson, _ = json.Marshal(entry.MetaData()) } - id := uuid.New().String() - _, err = r.db.Exec("INSERT INTO entries (id, type, published_at, meta_data) VALUES (?, ?, ?, ?)", id, t, publishedAt, metaDataJson) - entry.SetID(id) - entry.SetPublishedAt(publishedAt) - entry.SetMetaData(metaData) + if entry.ID() == "" { + entry.SetID(uuid.New().String()) + } + + _, err = r.db.Exec("INSERT INTO entries (id, type, published_at, meta_data) VALUES (?, ?, ?, ?)", entry.ID(), t, entry.PublishedAt(), metaDataJson) return err } diff --git a/infra/entry_repository_test.go b/infra/entry_repository_test.go index 949c9b2..a6178c1 100644 --- a/infra/entry_repository_test.go +++ b/infra/entry_repository_test.go @@ -24,11 +24,13 @@ func TestRepoCreate(t *testing.T) { entry := &test.MockEntry{} now := time.Now() - err := repo.Create(entry, &now, &test.MockEntryMetaData{ + entry.SetPublishedAt(&now) + entry.SetMetaData(&test.MockEntryMetaData{ Str: "str", Number: 1, Date: now, }) + err := repo.Create(entry) require.NoError(t, err) entry2, err := repo.FindById(entry.ID()) @@ -48,11 +50,13 @@ func TestRepoDelete(t *testing.T) { entry := &test.MockEntry{} now := time.Now() - err := repo.Create(entry, &now, &test.MockEntryMetaData{ + entry.SetPublishedAt(&now) + entry.SetMetaData(&test.MockEntryMetaData{ Str: "str", Number: 1, Date: now, }) + err := repo.Create(entry) require.NoError(t, err) err = repo.Delete(entry) @@ -67,21 +71,26 @@ func TestRepoFindAll(t *testing.T) { entry := &test.MockEntry{} now := time.Now() - err := repo.Create(entry, &now, &test.MockEntryMetaData{ + entry.SetPublishedAt(&now) + entry.SetMetaData(&test.MockEntryMetaData{ Str: "str", Number: 1, Date: now, }) + err := repo.Create(entry) require.NoError(t, err) entry2 := &test.MockEntry{} now2 := time.Now() - err = repo.Create(entry2, &now2, &test.MockEntryMetaData{ + entry2.SetPublishedAt(&now2) + entry2.SetMetaData(&test.MockEntryMetaData{ Str: "str", Number: 1, Date: now, }) + + err = repo.Create(entry2) require.NoError(t, err) entries, err := repo.FindAll(nil) @@ -103,20 +112,24 @@ func TestRepoUpdate(t *testing.T) { entry := &test.MockEntry{} now := time.Now() - err := repo.Create(entry, &now, &test.MockEntryMetaData{ + entry.SetPublishedAt(&now) + entry.SetMetaData(&test.MockEntryMetaData{ Str: "str", Number: 1, Date: now, }) + err := repo.Create(entry) require.NoError(t, err) entry2 := &test.MockEntry{} now2 := time.Now() - err = repo.Create(entry2, &now2, &test.MockEntryMetaData{ + entry2.SetPublishedAt(&now2) + entry2.SetMetaData(&test.MockEntryMetaData{ Str: "str2", Number: 2, Date: now2, }) + err = repo.Create(entry2) require.NoError(t, err) err = repo.Update(entry2) require.NoError(t, err) @@ -137,20 +150,25 @@ func TestRepoNoSideEffect(t *testing.T) { entry1 := &test.MockEntry{} now1 := time.Now() - err := repo.Create(entry1, &now1, &test.MockEntryMetaData{ + entry1.SetPublishedAt(&now1) + entry1.SetMetaData(&test.MockEntryMetaData{ Str: "1", Number: 1, Date: now1, }) + + err := repo.Create(entry1) require.NoError(t, err) entry2 := &test.MockEntry{} now2 := time.Now() - err = repo.Create(entry2, &now2, &test.MockEntryMetaData{ + entry2.SetPublishedAt(&now2) + entry2.SetMetaData(&test.MockEntryMetaData{ Str: "2", Number: 2, Date: now2, }) + err = repo.Create(entry2) require.NoError(t, err) r1, err := repo.FindById(entry1.ID()) diff --git a/web/editor_handler.go b/web/editor_handler.go index 6d87f0e..bb75e94 100644 --- a/web/editor_handler.go +++ b/web/editor_handler.go @@ -69,7 +69,9 @@ func (h *EditorHandler) HandlePost(c *fiber.Ctx) error { // create entry now := time.Now() - err = h.entrySvc.Create(entry, &now, entry.MetaData()) + entry.SetPublishedAt(&now) + + err = h.entrySvc.Create(entry) if err != nil { return err }