From 4b9583b08ff83dbbdae6541fd897d4bb790cd6ca Mon Sep 17 00:00:00 2001 From: Kasper Juul Hermansen Date: Tue, 21 Dec 2021 23:05:00 +0100 Subject: [PATCH] Add yt-downloader --- .idea/vcs.xml | 6 + api/go.mod | 3 + api/go.sum | 49 +++++++ .../app/api/download/request_download.go | 7 +- api/internal/app/api/download/routes.go | 6 +- api/internal/app/router/router.go | 42 +++++- api/internal/core/entities/download.go | 6 +- .../core/ports/download_request/in_memory.go | 110 --------------- .../download_request/in_memory/repository.go | 56 ++++++++ .../{download_request.go => repository.go} | 9 -- .../core/ports/downloader/downloader.go | 5 + .../core/ports/downloader/yt_downloader/yt.go | 130 ++++++++++++++++++ .../core/services/download/background.go | 7 + .../services/download/default/background.go | 55 ++++++++ .../core/services/download/default/service.go | 53 +++++++ .../core/services/download/service.go | 8 ++ api/pkg/files/move.go | 31 +++++ 17 files changed, 449 insertions(+), 134 deletions(-) create mode 100644 .idea/vcs.xml delete mode 100644 api/internal/core/ports/download_request/in_memory.go create mode 100644 api/internal/core/ports/download_request/in_memory/repository.go rename api/internal/core/ports/download_request/{download_request.go => repository.go} (54%) create mode 100644 api/internal/core/ports/downloader/downloader.go create mode 100644 api/internal/core/ports/downloader/yt_downloader/yt.go create mode 100644 api/internal/core/services/download/background.go create mode 100644 api/internal/core/services/download/default/background.go create mode 100644 api/internal/core/services/download/default/service.go create mode 100644 api/internal/core/services/download/service.go create mode 100644 api/pkg/files/move.go diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/api/go.mod b/api/go.mod index a838522..516941c 100644 --- a/api/go.mod +++ b/api/go.mod @@ -6,4 +6,7 @@ require ( github.com/go-chi/chi v1.5.4 // indirect github.com/go-chi/render v1.0.1 // indirect github.com/google/uuid v1.3.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + go.uber.org/zap v1.19.1 // indirect ) diff --git a/api/go.sum b/api/go.sum index 7606d92..fa5f791 100644 --- a/api/go.sum +++ b/api/go.sum @@ -1,6 +1,55 @@ +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/api/internal/app/api/download/request_download.go b/api/internal/app/api/download/request_download.go index a5b5123..b462fa6 100644 --- a/api/internal/app/api/download/request_download.go +++ b/api/internal/app/api/download/request_download.go @@ -9,8 +9,7 @@ import ( ) type requestDownloadRequest struct { - Provider string `json:"provider"` - Link string `json:"link"` + Link string `json:"link"` } type requestDownloadResponse struct { @@ -22,7 +21,7 @@ func (_ requestDownloadResponse) Render(_ http.ResponseWriter, _ *http.Request) } func (dr *requestDownloadRequest) Bind(r *http.Request) error { - if dr.Link == "" || dr.Provider == "" { + if dr.Link == "" { return errors.New("missing required download request field") } @@ -36,7 +35,7 @@ func (a *api) requestDownload(w http.ResponseWriter, r *http.Request) { return } - download, err := a.drService.Schedule(data.Provider, data.Link) + download, err := a.drService.Schedule(data.Link) if err != nil { _ = render.Render(w, r, responses.ErrInvalidRequest(err)) return diff --git a/api/internal/app/api/download/routes.go b/api/internal/app/api/download/routes.go index f68ba63..22bd065 100644 --- a/api/internal/app/api/download/routes.go +++ b/api/internal/app/api/download/routes.go @@ -1,15 +1,15 @@ package download import ( - "downloader/internal/core/ports/download_request" + "downloader/internal/core/services/download" "github.com/go-chi/chi" ) type api struct { - drService download_request.Service + drService download.Service } -func New(service download_request.Service) *api { +func New(service download.Service) *api { return &api{drService: service} } diff --git a/api/internal/app/router/router.go b/api/internal/app/router/router.go index 4094aa6..1cd6be4 100644 --- a/api/internal/app/router/router.go +++ b/api/internal/app/router/router.go @@ -2,11 +2,16 @@ package router import ( "downloader/internal/app/api/download" - "downloader/internal/core/ports/download_request" + "downloader/internal/core/ports/download_request/in_memory" + "downloader/internal/core/ports/downloader/yt_downloader" + "downloader/internal/core/services/download/default" "downloader/pkg/common/uuid" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" "net/http" + "os" "time" ) @@ -49,11 +54,40 @@ func (router *router) setupRoutes() *router { } func setupDownloadRoute(router *router) { - drRepository := download_request.NewInMemoryRepository() - drBackgroundService := download_request.NewLocalBackgroundService(drRepository) + sugaredLogger := setupLogger() + + drRepository := in_memory.NewInMemoryRepository(sugaredLogger) + downloader := yt_downloader.New(sugaredLogger) + drBackgroundService := _default.NewLocalBackgroundService(drRepository, sugaredLogger, downloader) gen := uuid.New() - drService := download_request.NewLocalService(drRepository, gen, drBackgroundService) + drService := _default.NewLocalService(drRepository, gen, drBackgroundService, sugaredLogger) downloadApi := download.New(drService) downloadApi.SetupDownloadApi(router.internalRouter) } + +func setupLogger() *zap.SugaredLogger { + consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()) + jsonEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()) + + consoleDebugging := zapcore.Lock(os.Stdout) + lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl < zapcore.ErrorLevel + }) + highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + return lvl >= zapcore.ErrorLevel + }) + consoleErrors := zapcore.Lock(os.Stderr) + core := zapcore.NewTee( + zapcore.NewCore(jsonEncoder, consoleErrors, highPriority), + zapcore.NewCore(jsonEncoder, consoleDebugging, highPriority), + zapcore.NewCore(consoleEncoder, consoleErrors, highPriority), + zapcore.NewCore(consoleEncoder, consoleDebugging, lowPriority), + ) + logger := zap.New(core) + defer func(logger *zap.Logger) { + _ = logger.Sync() + }(logger) + sugaredLogger := logger.Sugar() + return sugaredLogger +} diff --git a/api/internal/core/entities/download.go b/api/internal/core/entities/download.go index d65980a..a18f7d7 100644 --- a/api/internal/core/entities/download.go +++ b/api/internal/core/entities/download.go @@ -9,12 +9,11 @@ type Download struct { ID string `json:"id"` Status string `json:"status"` Link string `json:"link"` - Data string `json:"data"` } -func NewDownload(link string, data string) func(uuidGen uuid.Gen) (*Download, error) { +func NewDownload(link string) func(uuidGen uuid.Gen) (*Download, error) { return func(uuidGen uuid.Gen) (*Download, error) { - if link == "" || data == "" { + if link == "" { return nil, errors.New("A field was not valid") } @@ -22,7 +21,6 @@ func NewDownload(link string, data string) func(uuidGen uuid.Gen) (*Download, er ID: uuidGen.Create(), Status: "scheduled", Link: link, - Data: data, }, nil } } diff --git a/api/internal/core/ports/download_request/in_memory.go b/api/internal/core/ports/download_request/in_memory.go deleted file mode 100644 index c440376..0000000 --- a/api/internal/core/ports/download_request/in_memory.go +++ /dev/null @@ -1,110 +0,0 @@ -package download_request - -import ( - "downloader/internal/core/entities" - "downloader/pkg/common/uuid" - "errors" - "fmt" - "time" -) - -// Repository -type inMemoryRepository struct { - collection map[string]*entities.Download -} - -func NewInMemoryRepository() Repository { - return &inMemoryRepository{collection: make(map[string]*entities.Download)} -} - -func (i *inMemoryRepository) Create(download *entities.Download) (*entities.Download, error) { - if doc := i.collection[download.ID]; doc != nil { - return nil, errors.New("download request already exists") - } - - i.collection[download.ID] = download - - return download, nil -} - -func (i *inMemoryRepository) Update(download *entities.Download) error { - - if doc := i.collection[download.ID]; doc == nil { - return errors.New("download request doesn't exist exists") - } - - i.collection[download.ID] = download - - return nil -} - -func (i *inMemoryRepository) GetById(id string) (*entities.Download, error) { - if download := i.collection[id]; download != nil { - return download, nil - } else { - return nil, errors.New("download was not found in the database") - } -} - -// Service -type localService struct { - repository Repository - uuidGen uuid.Gen - BackgroundService BackgroundService -} - -func NewLocalService(repository Repository, uuidGen uuid.Gen, backgroundService BackgroundService) Service { - return &localService{ - repository: repository, - uuidGen: uuidGen, - BackgroundService: backgroundService, - } -} - -func (l *localService) Schedule(provider string, link string) (*entities.Download, error) { - download, err := entities.NewDownload(link, provider)(l.uuidGen) - if err != nil { - return nil, err - } - - persistedDownloadRequest, uploadErr := l.repository.Create(download) - if uploadErr != nil { - return nil, uploadErr - } - - err = l.BackgroundService.Run(persistedDownloadRequest) - if err != nil { - return nil, err - } - - return persistedDownloadRequest, nil -} - -func (l *localService) Get(id string) (*entities.Download, error) { - return l.repository.GetById(id) -} - -// Background service -type localBackgroundService struct { - repository Repository -} - -func NewLocalBackgroundService(repository Repository) BackgroundService { - return &localBackgroundService{repository: repository} -} - -func (l localBackgroundService) Run(download *entities.Download) error { - go func() { - time.Sleep(time.Second * 5) - download.Status = "done" - err := l.repository.Update(download) - if err != nil { - fmt.Printf("Download request: %s failed\n", download.ID) - panic(err) - } else { - fmt.Printf("Download request: %s done\n", download.ID) - } - }() - - return nil -} diff --git a/api/internal/core/ports/download_request/in_memory/repository.go b/api/internal/core/ports/download_request/in_memory/repository.go new file mode 100644 index 0000000..5f8202e --- /dev/null +++ b/api/internal/core/ports/download_request/in_memory/repository.go @@ -0,0 +1,56 @@ +package in_memory + +import ( + "downloader/internal/core/entities" + "downloader/internal/core/ports/download_request" + "errors" + "go.uber.org/zap" +) + +type inMemoryRepository struct { + collection map[string]*entities.Download + logger *zap.SugaredLogger +} + +func NewInMemoryRepository(logger *zap.SugaredLogger) download_request.Repository { + return &inMemoryRepository{collection: make(map[string]*entities.Download), logger: logger} +} + +func (i *inMemoryRepository) Create(download *entities.Download) (*entities.Download, error) { + logger := i.logger.With("downloadId", download.ID) + + if doc := i.collection[download.ID]; doc != nil { + logger.Warn("create: download already exists") + return nil, errors.New("download request already exists") + } + + i.collection[download.ID] = download + logger.Info("added download to database") + + return download, nil +} + +func (i *inMemoryRepository) Update(download *entities.Download) error { + logger := i.logger.With("downloadId", download.ID) + + if doc := i.collection[download.ID]; doc == nil { + logger.Warn("update: download doesnt exist") + return errors.New("download request doesn't exist exists") + } + + i.collection[download.ID] = download + logger.Info("updated download to database") + + return nil +} + +func (i *inMemoryRepository) GetById(id string) (*entities.Download, error) { + logger := i.logger.With("downloadId", id) + + if download := i.collection[id]; download != nil { + return download, nil + } else { + logger.Warn("download was not found in the database") + return nil, errors.New("download was not found in the database") + } +} diff --git a/api/internal/core/ports/download_request/download_request.go b/api/internal/core/ports/download_request/repository.go similarity index 54% rename from api/internal/core/ports/download_request/download_request.go rename to api/internal/core/ports/download_request/repository.go index c0cd802..d727e36 100644 --- a/api/internal/core/ports/download_request/download_request.go +++ b/api/internal/core/ports/download_request/repository.go @@ -2,17 +2,8 @@ package download_request import "downloader/internal/core/entities" -type Service interface { - Schedule(provider string, link string) (*entities.Download, error) - Get(id string) (*entities.Download, error) -} - type Repository interface { Create(download *entities.Download) (*entities.Download, error) GetById(id string) (*entities.Download, error) Update(download *entities.Download) error } - -type BackgroundService interface { - Run(download *entities.Download) error -} diff --git a/api/internal/core/ports/downloader/downloader.go b/api/internal/core/ports/downloader/downloader.go new file mode 100644 index 0000000..8b3bada --- /dev/null +++ b/api/internal/core/ports/downloader/downloader.go @@ -0,0 +1,5 @@ +package downloader + +type Downloader interface { + Download(link string, updateEvent func(progress string)) error +} diff --git a/api/internal/core/ports/downloader/yt_downloader/yt.go b/api/internal/core/ports/downloader/yt_downloader/yt.go new file mode 100644 index 0000000..09726c9 --- /dev/null +++ b/api/internal/core/ports/downloader/yt_downloader/yt.go @@ -0,0 +1,130 @@ +package yt_downloader + +import ( + "crypto/sha256" + "downloader/internal/core/ports/downloader" + "downloader/pkg/files" + "fmt" + "go.uber.org/zap" + "io" + "io/fs" + "log" + "os" + "os/exec" + "regexp" + "strings" + "time" +) + +type YtDownloader struct { + outputDirectory string + tempDirectory string + checkFrequencyMs time.Duration + logger *zap.SugaredLogger +} + +func New(logger *zap.SugaredLogger) downloader.Downloader { + return &YtDownloader{ + outputDirectory: "/home/hermansen/Downloads/yt", + tempDirectory: "/tmp/downloader", + checkFrequencyMs: 5000, + logger: logger, + } +} + +func init() { + _, err := exec.Command("youtube-dl", "--version").Output() + if err != nil { + log.Fatal("Youtube download (youtube-dl) isn't installed on the device") + } +} + +func (y *YtDownloader) Download(link string, updateEvent func(progress string)) error { + baseDir := fmt.Sprintf("%s/%x", + y.tempDirectory, + sha256.Sum256([]byte(link))) + err := os.MkdirAll(baseDir, os.ModePerm) + err = os.MkdirAll(y.outputDirectory, os.ModePerm) + if err != nil { + y.logger.Error(err) + return err + } + + filePath := fmt.Sprintf("%s/%s", baseDir, "%(title)s-%(id)s.%(ext)s") + + command := exec.Command("youtube-dl", + "-R 3", + "-o", + filePath, + link, + ) + + var stdout io.ReadCloser + stdout, err = command.StdoutPipe() + + go func() { + for true { + bytes := make([]byte, 1024) + _, err = stdout.Read(bytes) + if err != nil { + return + } + output := string(bytes) + + compile := regexp.MustCompile(`[a-z\[\]\s]+([\d\.]+).[a-z\s]+([\d\.]+)([a-zA-Z]+)[a-z\s]+`) + if err != nil { + y.logger.Error(err) + return + } + + res := compile.FindAllStringSubmatch(output, -1) + if len(res) != 0 && strings.Contains(res[0][0], "download") && len(res[0]) >= 2 { + progress := res[0][1] + + y.logger.Debugw("progress", + "percentage", progress, + "link", link) + updateEvent(progress) + } + + time.Sleep(time.Millisecond * y.checkFrequencyMs) + } + }() + + err = command.Start() + + if err != nil { + y.logger.Warn(err) + return err + } + + err = command.Wait() + if err != nil { + return err + } + + var dir []fs.DirEntry + dir, err = os.ReadDir(baseDir) + if err != nil { + y.logger.Error("Could not read directory") + return err + } + + for _, fileInfo := range dir { + oldPath := fmt.Sprintf("%s/%s", baseDir, fileInfo.Name()) + newPath := fmt.Sprintf("%s/%s", y.outputDirectory, fileInfo.Name()) + + if err := files.MoveFile(oldPath, newPath); err != nil { + return err + } else { + y.logger.Infow("moved file", + "fileName", fileInfo.Name()) + if err := os.Remove(baseDir); err != nil { + y.logger.Warn("could not cleanup", + "path", oldPath) + } + } + } + + return nil +} diff --git a/api/internal/core/services/download/background.go b/api/internal/core/services/download/background.go new file mode 100644 index 0000000..6fd6a57 --- /dev/null +++ b/api/internal/core/services/download/background.go @@ -0,0 +1,7 @@ +package download + +import "downloader/internal/core/entities" + +type BackgroundService interface { + Run(download *entities.Download) error +} diff --git a/api/internal/core/services/download/default/background.go b/api/internal/core/services/download/default/background.go new file mode 100644 index 0000000..2a090a8 --- /dev/null +++ b/api/internal/core/services/download/default/background.go @@ -0,0 +1,55 @@ +package _default + +import ( + "downloader/internal/core/entities" + "downloader/internal/core/ports/download_request" + "downloader/internal/core/ports/downloader" + "downloader/internal/core/services/download" + "fmt" + "go.uber.org/zap" +) + +type localBackgroundService struct { + repository download_request.Repository + logger *zap.SugaredLogger + downloader downloader.Downloader +} + +func NewLocalBackgroundService(repository download_request.Repository, logger *zap.SugaredLogger, downloader downloader.Downloader) download.BackgroundService { + return &localBackgroundService{repository: repository, logger: logger, downloader: downloader} +} + +func (l localBackgroundService) Run(download *entities.Download) error { + logger := l.logger.With("downloadId", download.ID) + + go func() { + download.Status = "started" + _ = l.repository.Update(download) + + err := l.downloader.Download(download.Link, func(progress string) { + download.Status = fmt.Sprintf("in-progress: %s", progress) + _ = l.repository.Update(download) + }) + download.Status = "done" + + if err != nil { + logger.Error(err) + download.Status = "failed" + } + + err = l.repository.Update(download) + if err != nil { + logger.Errorw("download request failed", + "downloadLink", download.Link) + download.Status = "failed" + + if updateErr := l.repository.Update(download); updateErr != nil { + panic(updateErr) + } + } else { + logger.Info("download request done") + } + }() + + return nil +} diff --git a/api/internal/core/services/download/default/service.go b/api/internal/core/services/download/default/service.go new file mode 100644 index 0000000..d9386b9 --- /dev/null +++ b/api/internal/core/services/download/default/service.go @@ -0,0 +1,53 @@ +package _default + +import ( + "downloader/internal/core/entities" + "downloader/internal/core/ports/download_request" + "downloader/internal/core/services/download" + "downloader/pkg/common/uuid" + "go.uber.org/zap" +) + +type localService struct { + repository download_request.Repository + uuidGen uuid.Gen + BackgroundService download.BackgroundService + logger *zap.SugaredLogger +} + +func NewLocalService(repository download_request.Repository, uuidGen uuid.Gen, backgroundService download.BackgroundService, logger *zap.SugaredLogger) download.Service { + return &localService{ + repository: repository, + uuidGen: uuidGen, + BackgroundService: backgroundService, + logger: logger, + } +} + +func (l *localService) Schedule(link string) (*entities.Download, error) { + download, err := entities.NewDownload(link)(l.uuidGen) + if err != nil { + l.logger.Warn("Could not parse download") + return nil, err + } + + logger := l.logger.With("downloadId", download.ID) + + persistedDownloadRequest, uploadErr := l.repository.Create(download) + if uploadErr != nil { + logger.Error("failed to insert download request") + return nil, uploadErr + } + + err = l.BackgroundService.Run(persistedDownloadRequest) + if err != nil { + logger.Error("failed to run download request") + return nil, err + } + + return persistedDownloadRequest, nil +} + +func (l *localService) Get(id string) (*entities.Download, error) { + return l.repository.GetById(id) +} diff --git a/api/internal/core/services/download/service.go b/api/internal/core/services/download/service.go new file mode 100644 index 0000000..0e07126 --- /dev/null +++ b/api/internal/core/services/download/service.go @@ -0,0 +1,8 @@ +package download + +import "downloader/internal/core/entities" + +type Service interface { + Schedule(link string) (*entities.Download, error) + Get(id string) (*entities.Download, error) +} diff --git a/api/pkg/files/move.go b/api/pkg/files/move.go new file mode 100644 index 0000000..b7e260c --- /dev/null +++ b/api/pkg/files/move.go @@ -0,0 +1,31 @@ +package files + +import ( + "fmt" + "io" + "os" +) + +func MoveFile(sourcePath, destPath string) error { + inputFile, err := os.Open(sourcePath) + if err != nil { + return fmt.Errorf("couldn't open source file: %s", err) + } + outputFile, err := os.Create(destPath) + if err != nil { + inputFile.Close() + return fmt.Errorf("couldn't open dest file: %s", err) + } + defer outputFile.Close() + _, err = io.Copy(outputFile, inputFile) + inputFile.Close() + if err != nil { + return fmt.Errorf("writing to output file failed: %s", err) + } + // The copy was successful, so now delete the original file + err = os.Remove(sourcePath) + if err != nil { + return fmt.Errorf("failed removing original file: %s", err) + } + return nil +}