diff --git a/api/cmd/api/main.go b/api/cmd/api/main.go index acf460c..a707e9b 100644 --- a/api/cmd/api/main.go +++ b/api/cmd/api/main.go @@ -1,8 +1,13 @@ package main -import router "downloader/internal/app/router" +import ( + "downloader/internal/app/configuration" + "downloader/internal/app/router" +) func main() { + configuration.SetupDevelopmentConfigProvider() + router. NewRouter(). Run() diff --git a/api/config.development.cfg b/api/config.development.cfg new file mode 100644 index 0000000..77d865b --- /dev/null +++ b/api/config.development.cfg @@ -0,0 +1,6 @@ +- key: download_output + value: /home/kjuulh/Downloads/downloader +- key: download_tmp_output + value: /tmp/downloader +- key: download_frequency_update_ms + value: 1000 \ No newline at end of file diff --git a/api/features.cfg b/api/features.cfg new file mode 100644 index 0000000..5d0e080 --- /dev/null +++ b/api/features.cfg @@ -0,0 +1,2 @@ +- key: download_progress_update + value: true diff --git a/api/go.mod b/api/go.mod index 516941c..73b9a32 100644 --- a/api/go.mod +++ b/api/go.mod @@ -3,10 +3,13 @@ module downloader go 1.17 require ( + github.com/ghodss/yaml v1.0.0 // indirect github.com/go-chi/chi v1.5.4 // indirect github.com/go-chi/render v1.0.1 // indirect github.com/google/uuid v1.3.0 // indirect + github.com/ovh/configstore v0.5.2 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/api/go.sum b/api/go.sum index fa5f791..86439ce 100644 --- a/api/go.sum +++ b/api/go.sum @@ -1,6 +1,8 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= @@ -10,10 +12,13 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/ovh/configstore v0.5.2 h1:VgraYqXx35W3ESBuGBzKdaa23FOVjJ7u1aY/zjeYcZw= +github.com/ovh/configstore v0.5.2/go.mod h1:krvoiDcrESKjEOz6AnKD8EbCmqHHQyOzEJNhAIhB4+Y= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= @@ -51,5 +56,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/api/internal/app/configuration/config.go b/api/internal/app/configuration/config.go new file mode 100644 index 0000000..c03c99c --- /dev/null +++ b/api/internal/app/configuration/config.go @@ -0,0 +1,14 @@ +package configuration + +import "github.com/ovh/configstore" + +func SetupDevelopmentConfigProvider() { + configstore.Env("DOWNLOADER_") + configstore.File("config.development.cfg") + configstore.File("features.cfg") +} + +func SetupProductionConfigProvider() { + configstore.Env("DOWNLOADER_") + configstore.File("config.cfg") +} diff --git a/api/internal/app/infrastructure/logger/logger.go b/api/internal/app/infrastructure/logger/logger.go new file mode 100644 index 0000000..06645f1 --- /dev/null +++ b/api/internal/app/infrastructure/logger/logger.go @@ -0,0 +1,33 @@ +package logger + +import ( + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "os" +) + +func New() *zap.SugaredLogger { + consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) + jsonEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()) + + consoleDebugging := zapcore.Lock(os.Stdout) + lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl < zapcore.ErrorLevel + }) + highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl >= zapcore.ErrorLevel + }) + consoleErrors := zapcore.Lock(os.Stderr) + core := zapcore.NewTee( + zapcore.NewCore(jsonEncoder, consoleErrors, highPriority), + zapcore.NewCore(jsonEncoder, consoleDebugging, highPriority), + zapcore.NewCore(consoleEncoder, consoleErrors, highPriority), + zapcore.NewCore(consoleEncoder, consoleDebugging, lowPriority), + ) + logger := zap.New(core) + defer func(logger *zap.Logger) { + _ = logger.Sync() + }(logger) + sugaredLogger := logger.Sugar() + return sugaredLogger +} diff --git a/api/internal/app/router/router.go b/api/internal/app/router/router.go index 1cd6be4..d9901d1 100644 --- a/api/internal/app/router/router.go +++ b/api/internal/app/router/router.go @@ -2,16 +2,20 @@ package router import ( "downloader/internal/app/api/download" + "downloader/internal/app/infrastructure/logger" "downloader/internal/core/ports/download_request/in_memory" - "downloader/internal/core/ports/downloader/yt_downloader" + "downloader/internal/core/ports/downloadhandler" + "downloader/internal/core/ports/filehandler/mover/local" + "downloader/internal/core/ports/fileorchestrator" + "downloader/internal/core/ports/fileorchestrator/destinationhandler" + "downloader/internal/core/ports/fileorchestrator/sourcehandler" "downloader/internal/core/services/download/default" + "downloader/internal/core/services/download/handlers" + "downloader/internal/core/services/downloader" "downloader/pkg/common/uuid" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" "net/http" - "os" "time" ) @@ -54,40 +58,21 @@ func (router *router) setupRoutes() *router { } func setupDownloadRoute(router *router) { - sugaredLogger := setupLogger() + newLogger := logger.New() + sourceHandler := sourcehandler.New() + mover := local.New(newLogger) + destinationHandler := destinationhandler.New(mover) + fileOrchestrator := fileorchestrator.New(newLogger, sourceHandler, destinationHandler) - drRepository := in_memory.NewInMemoryRepository(sugaredLogger) - downloader := yt_downloader.New(sugaredLogger) - drBackgroundService := _default.NewLocalBackgroundService(drRepository, sugaredLogger, downloader) + drRepository := in_memory.NewInMemoryRepository(newLogger) + //dlHandler := downloadhandler.NewYoutubeDlDownloader(newLogger) + dlHandler := downloadhandler.NewYtDlpDownloader(newLogger) + ondlHandler := handlers.New(drRepository, newLogger) + dwnloader := downloader.New(newLogger, fileOrchestrator, dlHandler, ondlHandler) + drBackgroundService := _default.NewLocalBackgroundService(drRepository, newLogger, dwnloader) gen := uuid.New() + drService := _default.NewLocalService(drRepository, gen, drBackgroundService, newLogger) - drService := _default.NewLocalService(drRepository, gen, drBackgroundService, sugaredLogger) downloadApi := download.New(drService) downloadApi.SetupDownloadApi(router.internalRouter) } - -func setupLogger() *zap.SugaredLogger { - consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) - jsonEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()) - - consoleDebugging := zapcore.Lock(os.Stdout) - lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { - return lvl < zapcore.ErrorLevel - }) - highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { - return lvl >= zapcore.ErrorLevel - }) - consoleErrors := zapcore.Lock(os.Stderr) - core := zapcore.NewTee( - zapcore.NewCore(jsonEncoder, consoleErrors, highPriority), - zapcore.NewCore(jsonEncoder, consoleDebugging, highPriority), - zapcore.NewCore(consoleEncoder, consoleErrors, highPriority), - zapcore.NewCore(consoleEncoder, consoleDebugging, lowPriority), - ) - logger := zap.New(core) - defer func(logger *zap.Logger) { - _ = logger.Sync() - }(logger) - sugaredLogger := logger.Sugar() - return sugaredLogger -} diff --git a/api/internal/core/ports/downloader/downloader.go b/api/internal/core/ports/downloader/downloader.go deleted file mode 100644 index 8b3bada..0000000 --- a/api/internal/core/ports/downloader/downloader.go +++ /dev/null @@ -1,5 +0,0 @@ -package downloader - -type Downloader interface { - Download(link string, updateEvent func(progress string)) error -} diff --git a/api/internal/core/ports/downloader/yt_downloader/yt.go b/api/internal/core/ports/downloader/yt_downloader/yt.go deleted file mode 100644 index 09726c9..0000000 --- a/api/internal/core/ports/downloader/yt_downloader/yt.go +++ /dev/null @@ -1,130 +0,0 @@ -package yt_downloader - -import ( - "crypto/sha256" - "downloader/internal/core/ports/downloader" - "downloader/pkg/files" - "fmt" - "go.uber.org/zap" - "io" - "io/fs" - "log" - "os" - "os/exec" - "regexp" - "strings" - "time" -) - -type YtDownloader struct { - outputDirectory string - tempDirectory string - checkFrequencyMs time.Duration - logger *zap.SugaredLogger -} - -func New(logger *zap.SugaredLogger) downloader.Downloader { - return &YtDownloader{ - outputDirectory: "/home/hermansen/Downloads/yt", - tempDirectory: "/tmp/downloader", - checkFrequencyMs: 5000, - logger: logger, - } -} - -func init() { - _, err := exec.Command("youtube-dl", "--version").Output() - if err != nil { - log.Fatal("Youtube download (youtube-dl) isn't installed on the device") - } -} - -func (y *YtDownloader) Download(link string, updateEvent func(progress string)) error { - baseDir := fmt.Sprintf("%s/%x", - y.tempDirectory, - sha256.Sum256([]byte(link))) - err := os.MkdirAll(baseDir, os.ModePerm) - err = os.MkdirAll(y.outputDirectory, os.ModePerm) - if err != nil { - y.logger.Error(err) - return err - } - - filePath := fmt.Sprintf("%s/%s", baseDir, "%(title)s-%(id)s.%(ext)s") - - command := exec.Command("youtube-dl", - "-R 3", - "-o", - filePath, - link, - ) - - var stdout io.ReadCloser - stdout, err = command.StdoutPipe() - - go func() { - for true { - bytes := make([]byte, 1024) - _, err = stdout.Read(bytes) - if err != nil { - return - } - output := string(bytes) - - compile := regexp.MustCompile(`[a-z\[\]\s]+([\d\.]+).[a-z\s]+([\d\.]+)([a-zA-Z]+)[a-z\s]+`) - if err != nil { - y.logger.Error(err) - return - } - - res := compile.FindAllStringSubmatch(output, -1) - if len(res) != 0 && strings.Contains(res[0][0], "download") && len(res[0]) >= 2 { - progress := res[0][1] - - y.logger.Debugw("progress", - "percentage", progress, - "link", link) - updateEvent(progress) - } - - time.Sleep(time.Millisecond * y.checkFrequencyMs) - } - }() - - err = command.Start() - - if err != nil { - y.logger.Warn(err) - return err - } - - err = command.Wait() - if err != nil { - return err - } - - var dir []fs.DirEntry - dir, err = os.ReadDir(baseDir) - if err != nil { - y.logger.Error("Could not read directory") - return err - } - - for _, fileInfo := range dir { - oldPath := fmt.Sprintf("%s/%s", baseDir, fileInfo.Name()) - newPath := fmt.Sprintf("%s/%s", y.outputDirectory, fileInfo.Name()) - - if err := files.MoveFile(oldPath, newPath); err != nil { - return err - } else { - y.logger.Infow("moved file", - "fileName", fileInfo.Name()) - if err := os.Remove(baseDir); err != nil { - y.logger.Warn("could not cleanup", - "path", oldPath) - } - } - } - - return nil -} diff --git a/api/internal/core/ports/downloadhandler/downloadhandler.go b/api/internal/core/ports/downloadhandler/downloadhandler.go new file mode 100644 index 0000000..46966e5 --- /dev/null +++ b/api/internal/core/ports/downloadhandler/downloadhandler.go @@ -0,0 +1,10 @@ +package downloadhandler + +type OnDownloadEventHandler interface { + OnTickEvent(downloadId string, progress string) +} + +type DownloadHandler interface { + OnProgress(eventHandler OnDownloadEventHandler) + Download(link string, outputDir string, downloadId string) error +} diff --git a/api/internal/core/ports/downloadhandler/youtube_dl.go b/api/internal/core/ports/downloadhandler/youtube_dl.go new file mode 100644 index 0000000..771c0be --- /dev/null +++ b/api/internal/core/ports/downloadhandler/youtube_dl.go @@ -0,0 +1,98 @@ +package downloadhandler + +import ( + "fmt" + "github.com/ovh/configstore" + "go.uber.org/zap" + "log" + "os/exec" + "regexp" + "strings" + "time" +) + +type YoutubeDlDownloader struct { + progressUpdate bool + checkFrequencyMs time.Duration + logger *zap.SugaredLogger + onDownloadEventHandler OnDownloadEventHandler +} + +func NewYoutubeDlDownloader(logger *zap.SugaredLogger) DownloadHandler { + var checkFrequencyMs int64 + var featureProgressUpdate bool + var err error + + checkFrequencyMs, err = configstore.GetItemValueInt("download_frequency_update_ms") + featureProgressUpdate, err = configstore.GetItemValueBool("download_progress_update") + + if err != nil { + panic(err) + } + + _, err = exec.Command("youtube-dl", "--version").Output() + if err != nil { + log.Fatal("Youtube download (youtube-dl) isn't installed on the device") + } + + return &YoutubeDlDownloader{ + checkFrequencyMs: time.Duration(checkFrequencyMs), + progressUpdate: featureProgressUpdate, + logger: logger, + onDownloadEventHandler: nil, + } +} + +func (y *YoutubeDlDownloader) OnProgress(eventHandler OnDownloadEventHandler) { + y.onDownloadEventHandler = eventHandler +} + +func (y *YoutubeDlDownloader) Download(link string, outputDir string, downloadId string) error { + filePath := fmt.Sprintf("%s/%s", outputDir, "%(title)s-%(id)s.%(ext)s") + command := exec.Command("youtube-dl", + "-R 3", + "-o", + filePath, + link, + ) + + stdout, err := command.StdoutPipe() + + if y.progressUpdate { + go func() { + for true { + bytes := make([]byte, 1024) + _, err = stdout.Read(bytes) + if err != nil { + return + } + output := string(bytes) + + if err = y.getParameters(output, link, downloadId); err != nil { + return + } + + time.Sleep(time.Millisecond * y.checkFrequencyMs) + } + }() + } + + return command.Run() +} + +func (y *YoutubeDlDownloader) getParameters(output string, link string, downloadId string) error { + compile := regexp.MustCompile(`[a-z\[\]\s]+([\d\.]+).[a-z\s]+([\d\.]+)([a-zA-Z]+)[a-z\s]+`) + + res := compile.FindAllStringSubmatch(output, -1) + if len(res) != 0 && strings.Contains(res[0][0], "download") && len(res[0]) >= 2 { + progress := res[0][1] + + y.logger.Debugw("progress", + "percentage", progress, + "link", link) + if y.onDownloadEventHandler != nil { + y.onDownloadEventHandler.OnTickEvent(downloadId, progress) + } + } + return nil +} diff --git a/api/internal/core/ports/downloadhandler/yt-dlp.go b/api/internal/core/ports/downloadhandler/yt-dlp.go new file mode 100644 index 0000000..bc53979 --- /dev/null +++ b/api/internal/core/ports/downloadhandler/yt-dlp.go @@ -0,0 +1,98 @@ +package downloadhandler + +import ( + "fmt" + "github.com/ovh/configstore" + "go.uber.org/zap" + "log" + "os/exec" + "regexp" + "strings" + "time" +) + +type YtDlpDownloader struct { + progressUpdate bool + checkFrequencyMs time.Duration + logger *zap.SugaredLogger + onDownloadEventHandler OnDownloadEventHandler +} + +func NewYtDlpDownloader(logger *zap.SugaredLogger) DownloadHandler { + var checkFrequencyMs int64 + var featureProgressUpdate bool + var err error + + checkFrequencyMs, err = configstore.GetItemValueInt("download_frequency_update_ms") + featureProgressUpdate, err = configstore.GetItemValueBool("download_progress_update") + + if err != nil { + panic(err) + } + + _, err = exec.Command("yt-dlp", "--version").Output() + if err != nil { + log.Fatal("Youtube download (youtube-dl) isn't installed on the device") + } + + return &YtDlpDownloader{ + checkFrequencyMs: time.Duration(checkFrequencyMs), + progressUpdate: featureProgressUpdate, + logger: logger, + onDownloadEventHandler: nil, + } +} + +func (y *YtDlpDownloader) OnProgress(eventHandler OnDownloadEventHandler) { + y.onDownloadEventHandler = eventHandler +} + +func (y *YtDlpDownloader) Download(link string, outputDir string, downloadId string) error { + filePath := fmt.Sprintf("%s/%s", outputDir, "%(title)s-%(id)s.%(ext)s") + command := exec.Command("yt-dlp", + "-R 3", + "-o", + filePath, + link, + ) + + stdout, err := command.StdoutPipe() + + if y.progressUpdate { + go func() { + for true { + bytes := make([]byte, 1024) + _, err = stdout.Read(bytes) + if err != nil { + return + } + output := string(bytes) + + if err = y.getParameters(output, link, downloadId); err != nil { + return + } + + time.Sleep(time.Millisecond * y.checkFrequencyMs) + } + }() + } + + return command.Run() +} + +func (y *YtDlpDownloader) getParameters(output string, link string, downloadId string) error { + compile := regexp.MustCompile(`[a-z\[\]\s]+([\d\.]+).[a-z\s\~]+([\d\.]+)([a-zA-Z]+)[a-z\s]+`) + + res := compile.FindAllStringSubmatch(output, -1) + if len(res) != 0 && strings.Contains(res[0][0], "download") && len(res[0]) >= 2 { + progress := res[0][1] + + y.logger.Debugw("progress", + "percentage", progress, + "link", link) + if y.onDownloadEventHandler != nil { + y.onDownloadEventHandler.OnTickEvent(downloadId, progress) + } + } + return nil +} diff --git a/api/internal/core/ports/filehandler/mover/local/mover.go b/api/internal/core/ports/filehandler/mover/local/mover.go new file mode 100644 index 0000000..0f9e2be --- /dev/null +++ b/api/internal/core/ports/filehandler/mover/local/mover.go @@ -0,0 +1,39 @@ +package local + +import ( + "downloader/internal/core/ports/filehandler/mover" + "downloader/pkg/files" + "fmt" + "go.uber.org/zap" + "os" +) + +type Mover struct { + logger *zap.SugaredLogger +} + +func New(logger *zap.SugaredLogger) mover.Mover { + return &Mover{logger: logger} +} + +func (m *Mover) Move(sourceDirectory string, destinationDirectory string) error { + dir, err := os.ReadDir(sourceDirectory) + if err != nil { + m.logger.Error("Could not read directory") + return err + } + + for _, fileInfo := range dir { + oldPath := fmt.Sprintf("%s/%s", sourceDirectory, fileInfo.Name()) + newPath := fmt.Sprintf("%s/%s", destinationDirectory, fileInfo.Name()) + + if err := files.MoveFile(oldPath, newPath); err != nil { + return err + } else { + m.logger.Infow("moved file", + "fileName", fileInfo.Name()) + } + } + + return nil +} diff --git a/api/internal/core/ports/filehandler/mover/mover.go b/api/internal/core/ports/filehandler/mover/mover.go new file mode 100644 index 0000000..6b65e7b --- /dev/null +++ b/api/internal/core/ports/filehandler/mover/mover.go @@ -0,0 +1,5 @@ +package mover + +type Mover interface { + Move(sourceDirectory string, destinationDirectory string) error +} diff --git a/api/internal/core/ports/fileorchestrator/destinationhandler/destinationhandler.go b/api/internal/core/ports/fileorchestrator/destinationhandler/destinationhandler.go new file mode 100644 index 0000000..b8c7838 --- /dev/null +++ b/api/internal/core/ports/fileorchestrator/destinationhandler/destinationhandler.go @@ -0,0 +1,36 @@ +package destinationhandler + +import ( + "downloader/internal/core/ports/filehandler/mover" + "github.com/ovh/configstore" + "os" +) + +type DestinationHandler interface { + Prepare() error + Move(sourcePath string) error +} + +type localDestinationHandler struct { + outputDirectory string + mover mover.Mover +} + +func New(mover mover.Mover) DestinationHandler { + outputDirectory, err := configstore.GetItemValue("download_output") + if err != nil { + panic(err) + } + return &localDestinationHandler{ + outputDirectory: outputDirectory, + mover: mover, + } +} + +func (l localDestinationHandler) Prepare() error { + return os.MkdirAll(l.outputDirectory, os.ModePerm) +} + +func (l localDestinationHandler) Move(sourcePath string) error { + return l.mover.Move(sourcePath, l.outputDirectory) +} diff --git a/api/internal/core/ports/fileorchestrator/fileorchestrator.go b/api/internal/core/ports/fileorchestrator/fileorchestrator.go new file mode 100644 index 0000000..82c9fa1 --- /dev/null +++ b/api/internal/core/ports/fileorchestrator/fileorchestrator.go @@ -0,0 +1,40 @@ +package fileorchestrator + +import ( + "downloader/internal/core/ports/fileorchestrator/destinationhandler" + "downloader/internal/core/ports/fileorchestrator/sourcehandler" + "go.uber.org/zap" +) + +type FileOrchestrator struct { + logger *zap.SugaredLogger + sourceHandler sourcehandler.SourceHandler + destinationHandler destinationhandler.DestinationHandler +} + +func New(logger *zap.SugaredLogger, sourceHandler sourcehandler.SourceHandler, destinationHandler destinationhandler.DestinationHandler) *FileOrchestrator { + return &FileOrchestrator{ + logger: logger, + sourceHandler: sourceHandler, + destinationHandler: destinationHandler, + } +} + +func (o *FileOrchestrator) Begin(link string) (string, error) { + basePath, err := o.sourceHandler.Prepare(link) + err = o.destinationHandler.Prepare() + + return basePath, err +} + +func (o *FileOrchestrator) CleanUp(sourceDirectory string) { + err := o.sourceHandler.CleanUp(sourceDirectory) + if err != nil { + o.logger.Errorw("could not cleanup", + "path", sourceDirectory) + } +} + +func (o *FileOrchestrator) Move(sourcePath string) error { + return o.destinationHandler.Move(sourcePath) +} diff --git a/api/internal/core/ports/fileorchestrator/sourcehandler/sourcehandler.go b/api/internal/core/ports/fileorchestrator/sourcehandler/sourcehandler.go new file mode 100644 index 0000000..b9228fe --- /dev/null +++ b/api/internal/core/ports/fileorchestrator/sourcehandler/sourcehandler.go @@ -0,0 +1,39 @@ +package sourcehandler + +import ( + "crypto/sha256" + "fmt" + "github.com/ovh/configstore" + "os" +) + +type SourceHandler interface { + Prepare(link string) (string, error) + CleanUp(sourcePath string) error +} + +type localSourceHandler struct { + outputTmpDirectory string +} + +func New() SourceHandler { + outputTmpDirectory, err := configstore.GetItemValue("download_tmp_output") + if err != nil { + panic(err) + } + return &localSourceHandler{ + outputTmpDirectory: outputTmpDirectory, + } +} + +func (l localSourceHandler) Prepare(link string) (string, error) { + path := fmt.Sprintf("%s/%x", + l.outputTmpDirectory, + sha256.Sum256([]byte(link))) + err := os.MkdirAll(path, os.ModePerm) + return path, err +} + +func (l localSourceHandler) CleanUp(sourcePath string) error { + return os.Remove(sourcePath) +} diff --git a/api/internal/core/services/download/default/background.go b/api/internal/core/services/download/default/background.go index 2a090a8..673fbed 100644 --- a/api/internal/core/services/download/default/background.go +++ b/api/internal/core/services/download/default/background.go @@ -3,9 +3,8 @@ package _default import ( "downloader/internal/core/entities" "downloader/internal/core/ports/download_request" - "downloader/internal/core/ports/downloader" "downloader/internal/core/services/download" - "fmt" + "downloader/internal/core/services/downloader" "go.uber.org/zap" ) @@ -26,10 +25,7 @@ func (l localBackgroundService) Run(download *entities.Download) error { download.Status = "started" _ = l.repository.Update(download) - err := l.downloader.Download(download.Link, func(progress string) { - download.Status = fmt.Sprintf("in-progress: %s", progress) - _ = l.repository.Update(download) - }) + err := l.downloader.Download(download.Link, download.ID) download.Status = "done" if err != nil { diff --git a/api/internal/core/services/download/handlers/onDownloadEventHandler.go b/api/internal/core/services/download/handlers/onDownloadEventHandler.go new file mode 100644 index 0000000..819795c --- /dev/null +++ b/api/internal/core/services/download/handlers/onDownloadEventHandler.go @@ -0,0 +1,32 @@ +package handlers + +import ( + "downloader/internal/core/ports/download_request" + "downloader/internal/core/ports/downloadhandler" + "fmt" + "go.uber.org/zap" +) + +type onDownloadEventHandler struct { + repository download_request.Repository + logger *zap.SugaredLogger +} + +func New(repository download_request.Repository, logger *zap.SugaredLogger) downloadhandler.OnDownloadEventHandler { + return &onDownloadEventHandler{ + repository: repository, + logger: logger, + } +} + +func (o *onDownloadEventHandler) OnTickEvent(downloadId string, progress string) { + download, err := o.repository.GetById(downloadId) + if err != nil { + o.logger.Warnw("could not finish updating progress as not download id available", + "downloadId", downloadId, + "progress", progress) + return + } + download.Status = fmt.Sprintf("in-progress: %s", progress) + _ = o.repository.Update(download) +} diff --git a/api/internal/core/services/downloader/downloader.go b/api/internal/core/services/downloader/downloader.go new file mode 100644 index 0000000..251aa3d --- /dev/null +++ b/api/internal/core/services/downloader/downloader.go @@ -0,0 +1,54 @@ +package downloader + +import ( + "downloader/internal/core/ports/downloadhandler" + "downloader/internal/core/ports/fileorchestrator" + "errors" + "go.uber.org/zap" +) + +type Downloader interface { + Download(link string, downloadId string) error +} + +type downloader struct { + logger *zap.SugaredLogger + downloadHandler downloadhandler.DownloadHandler + fileOrchestrator *fileorchestrator.FileOrchestrator + ondownloadhandler downloadhandler.OnDownloadEventHandler +} + +func New( + logger *zap.SugaredLogger, + orchestrator *fileorchestrator.FileOrchestrator, + downloadHandler downloadhandler.DownloadHandler, + downloadEventHandler downloadhandler.OnDownloadEventHandler, +) Downloader { + return &downloader{ + logger: logger, + downloadHandler: downloadHandler, + fileOrchestrator: orchestrator, + ondownloadhandler: downloadEventHandler, + } +} + +func (y *downloader) Download(link string, downloadId string) error { + basePath, err := y.fileOrchestrator.Begin(link) + if err != nil { + return errors.New("could not prepare for this link") + } + defer y.fileOrchestrator.CleanUp(basePath) + + y.downloadHandler.OnProgress(y.ondownloadhandler) + err = y.downloadHandler.Download(link, basePath, downloadId) + if err != nil { + return errors.New("could not download file") + } + + err = y.fileOrchestrator.Move(basePath) + if err != nil { + return errors.New("could not move to final destination") + } + + return nil +}