Refactored downloader
This commit is contained in:
parent
4b9583b08f
commit
23ca1168df
@ -1,8 +1,13 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import router "downloader/internal/app/router"
|
import (
|
||||||
|
"downloader/internal/app/configuration"
|
||||||
|
"downloader/internal/app/router"
|
||||||
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
configuration.SetupDevelopmentConfigProvider()
|
||||||
|
|
||||||
router.
|
router.
|
||||||
NewRouter().
|
NewRouter().
|
||||||
Run()
|
Run()
|
||||||
|
6
api/config.development.cfg
Normal file
6
api/config.development.cfg
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
- key: download_output
|
||||||
|
value: /home/kjuulh/Downloads/downloader
|
||||||
|
- key: download_tmp_output
|
||||||
|
value: /tmp/downloader
|
||||||
|
- key: download_frequency_update_ms
|
||||||
|
value: 1000
|
2
api/features.cfg
Normal file
2
api/features.cfg
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
- key: download_progress_update
|
||||||
|
value: true
|
@ -3,10 +3,13 @@ module downloader
|
|||||||
go 1.17
|
go 1.17
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/ghodss/yaml v1.0.0 // indirect
|
||||||
github.com/go-chi/chi v1.5.4 // indirect
|
github.com/go-chi/chi v1.5.4 // indirect
|
||||||
github.com/go-chi/render v1.0.1 // indirect
|
github.com/go-chi/render v1.0.1 // indirect
|
||||||
github.com/google/uuid v1.3.0 // indirect
|
github.com/google/uuid v1.3.0 // indirect
|
||||||
|
github.com/ovh/configstore v0.5.2 // indirect
|
||||||
go.uber.org/atomic v1.7.0 // indirect
|
go.uber.org/atomic v1.7.0 // indirect
|
||||||
go.uber.org/multierr v1.6.0 // indirect
|
go.uber.org/multierr v1.6.0 // indirect
|
||||||
go.uber.org/zap v1.19.1 // indirect
|
go.uber.org/zap v1.19.1 // indirect
|
||||||
|
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||||
)
|
)
|
||||||
|
@ -1,6 +1,8 @@
|
|||||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||||
|
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||||
github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs=
|
github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs=
|
||||||
github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg=
|
github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg=
|
||||||
github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
|
github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
|
||||||
@ -10,10 +12,13 @@ github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
|
|||||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
|
github.com/ovh/configstore v0.5.2 h1:VgraYqXx35W3ESBuGBzKdaa23FOVjJ7u1aY/zjeYcZw=
|
||||||
|
github.com/ovh/configstore v0.5.2/go.mod h1:krvoiDcrESKjEOz6AnKD8EbCmqHHQyOzEJNhAIhB4+Y=
|
||||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
|
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||||
@ -51,5 +56,7 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
|
|||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||||
|
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||||
|
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
14
api/internal/app/configuration/config.go
Normal file
14
api/internal/app/configuration/config.go
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
package configuration
|
||||||
|
|
||||||
|
import "github.com/ovh/configstore"
|
||||||
|
|
||||||
|
func SetupDevelopmentConfigProvider() {
|
||||||
|
configstore.Env("DOWNLOADER_")
|
||||||
|
configstore.File("config.development.cfg")
|
||||||
|
configstore.File("features.cfg")
|
||||||
|
}
|
||||||
|
|
||||||
|
func SetupProductionConfigProvider() {
|
||||||
|
configstore.Env("DOWNLOADER_")
|
||||||
|
configstore.File("config.cfg")
|
||||||
|
}
|
33
api/internal/app/infrastructure/logger/logger.go
Normal file
33
api/internal/app/infrastructure/logger/logger.go
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
package logger
|
||||||
|
|
||||||
|
import (
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"go.uber.org/zap/zapcore"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
func New() *zap.SugaredLogger {
|
||||||
|
consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())
|
||||||
|
jsonEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig())
|
||||||
|
|
||||||
|
consoleDebugging := zapcore.Lock(os.Stdout)
|
||||||
|
lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
|
||||||
|
return lvl < zapcore.ErrorLevel
|
||||||
|
})
|
||||||
|
highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
|
||||||
|
return lvl >= zapcore.ErrorLevel
|
||||||
|
})
|
||||||
|
consoleErrors := zapcore.Lock(os.Stderr)
|
||||||
|
core := zapcore.NewTee(
|
||||||
|
zapcore.NewCore(jsonEncoder, consoleErrors, highPriority),
|
||||||
|
zapcore.NewCore(jsonEncoder, consoleDebugging, highPriority),
|
||||||
|
zapcore.NewCore(consoleEncoder, consoleErrors, highPriority),
|
||||||
|
zapcore.NewCore(consoleEncoder, consoleDebugging, lowPriority),
|
||||||
|
)
|
||||||
|
logger := zap.New(core)
|
||||||
|
defer func(logger *zap.Logger) {
|
||||||
|
_ = logger.Sync()
|
||||||
|
}(logger)
|
||||||
|
sugaredLogger := logger.Sugar()
|
||||||
|
return sugaredLogger
|
||||||
|
}
|
@ -2,16 +2,20 @@ package router
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"downloader/internal/app/api/download"
|
"downloader/internal/app/api/download"
|
||||||
|
"downloader/internal/app/infrastructure/logger"
|
||||||
"downloader/internal/core/ports/download_request/in_memory"
|
"downloader/internal/core/ports/download_request/in_memory"
|
||||||
"downloader/internal/core/ports/downloader/yt_downloader"
|
"downloader/internal/core/ports/downloadhandler"
|
||||||
|
"downloader/internal/core/ports/filehandler/mover/local"
|
||||||
|
"downloader/internal/core/ports/fileorchestrator"
|
||||||
|
"downloader/internal/core/ports/fileorchestrator/destinationhandler"
|
||||||
|
"downloader/internal/core/ports/fileorchestrator/sourcehandler"
|
||||||
"downloader/internal/core/services/download/default"
|
"downloader/internal/core/services/download/default"
|
||||||
|
"downloader/internal/core/services/download/handlers"
|
||||||
|
"downloader/internal/core/services/downloader"
|
||||||
"downloader/pkg/common/uuid"
|
"downloader/pkg/common/uuid"
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"github.com/go-chi/chi/middleware"
|
"github.com/go-chi/chi/middleware"
|
||||||
"go.uber.org/zap"
|
|
||||||
"go.uber.org/zap/zapcore"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -54,40 +58,21 @@ func (router *router) setupRoutes() *router {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func setupDownloadRoute(router *router) {
|
func setupDownloadRoute(router *router) {
|
||||||
sugaredLogger := setupLogger()
|
newLogger := logger.New()
|
||||||
|
sourceHandler := sourcehandler.New()
|
||||||
|
mover := local.New(newLogger)
|
||||||
|
destinationHandler := destinationhandler.New(mover)
|
||||||
|
fileOrchestrator := fileorchestrator.New(newLogger, sourceHandler, destinationHandler)
|
||||||
|
|
||||||
drRepository := in_memory.NewInMemoryRepository(sugaredLogger)
|
drRepository := in_memory.NewInMemoryRepository(newLogger)
|
||||||
downloader := yt_downloader.New(sugaredLogger)
|
//dlHandler := downloadhandler.NewYoutubeDlDownloader(newLogger)
|
||||||
drBackgroundService := _default.NewLocalBackgroundService(drRepository, sugaredLogger, downloader)
|
dlHandler := downloadhandler.NewYtDlpDownloader(newLogger)
|
||||||
|
ondlHandler := handlers.New(drRepository, newLogger)
|
||||||
|
dwnloader := downloader.New(newLogger, fileOrchestrator, dlHandler, ondlHandler)
|
||||||
|
drBackgroundService := _default.NewLocalBackgroundService(drRepository, newLogger, dwnloader)
|
||||||
gen := uuid.New()
|
gen := uuid.New()
|
||||||
|
drService := _default.NewLocalService(drRepository, gen, drBackgroundService, newLogger)
|
||||||
|
|
||||||
drService := _default.NewLocalService(drRepository, gen, drBackgroundService, sugaredLogger)
|
|
||||||
downloadApi := download.New(drService)
|
downloadApi := download.New(drService)
|
||||||
downloadApi.SetupDownloadApi(router.internalRouter)
|
downloadApi.SetupDownloadApi(router.internalRouter)
|
||||||
}
|
}
|
||||||
|
|
||||||
func setupLogger() *zap.SugaredLogger {
|
|
||||||
consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())
|
|
||||||
jsonEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig())
|
|
||||||
|
|
||||||
consoleDebugging := zapcore.Lock(os.Stdout)
|
|
||||||
lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
|
|
||||||
return lvl < zapcore.ErrorLevel
|
|
||||||
})
|
|
||||||
highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
|
|
||||||
return lvl >= zapcore.ErrorLevel
|
|
||||||
})
|
|
||||||
consoleErrors := zapcore.Lock(os.Stderr)
|
|
||||||
core := zapcore.NewTee(
|
|
||||||
zapcore.NewCore(jsonEncoder, consoleErrors, highPriority),
|
|
||||||
zapcore.NewCore(jsonEncoder, consoleDebugging, highPriority),
|
|
||||||
zapcore.NewCore(consoleEncoder, consoleErrors, highPriority),
|
|
||||||
zapcore.NewCore(consoleEncoder, consoleDebugging, lowPriority),
|
|
||||||
)
|
|
||||||
logger := zap.New(core)
|
|
||||||
defer func(logger *zap.Logger) {
|
|
||||||
_ = logger.Sync()
|
|
||||||
}(logger)
|
|
||||||
sugaredLogger := logger.Sugar()
|
|
||||||
return sugaredLogger
|
|
||||||
}
|
|
||||||
|
@ -1,5 +0,0 @@
|
|||||||
package downloader
|
|
||||||
|
|
||||||
type Downloader interface {
|
|
||||||
Download(link string, updateEvent func(progress string)) error
|
|
||||||
}
|
|
@ -1,130 +0,0 @@
|
|||||||
package yt_downloader
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/sha256"
|
|
||||||
"downloader/internal/core/ports/downloader"
|
|
||||||
"downloader/pkg/files"
|
|
||||||
"fmt"
|
|
||||||
"go.uber.org/zap"
|
|
||||||
"io"
|
|
||||||
"io/fs"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
type YtDownloader struct {
|
|
||||||
outputDirectory string
|
|
||||||
tempDirectory string
|
|
||||||
checkFrequencyMs time.Duration
|
|
||||||
logger *zap.SugaredLogger
|
|
||||||
}
|
|
||||||
|
|
||||||
func New(logger *zap.SugaredLogger) downloader.Downloader {
|
|
||||||
return &YtDownloader{
|
|
||||||
outputDirectory: "/home/hermansen/Downloads/yt",
|
|
||||||
tempDirectory: "/tmp/downloader",
|
|
||||||
checkFrequencyMs: 5000,
|
|
||||||
logger: logger,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
_, err := exec.Command("youtube-dl", "--version").Output()
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal("Youtube download (youtube-dl) isn't installed on the device")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (y *YtDownloader) Download(link string, updateEvent func(progress string)) error {
|
|
||||||
baseDir := fmt.Sprintf("%s/%x",
|
|
||||||
y.tempDirectory,
|
|
||||||
sha256.Sum256([]byte(link)))
|
|
||||||
err := os.MkdirAll(baseDir, os.ModePerm)
|
|
||||||
err = os.MkdirAll(y.outputDirectory, os.ModePerm)
|
|
||||||
if err != nil {
|
|
||||||
y.logger.Error(err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
filePath := fmt.Sprintf("%s/%s", baseDir, "%(title)s-%(id)s.%(ext)s")
|
|
||||||
|
|
||||||
command := exec.Command("youtube-dl",
|
|
||||||
"-R 3",
|
|
||||||
"-o",
|
|
||||||
filePath,
|
|
||||||
link,
|
|
||||||
)
|
|
||||||
|
|
||||||
var stdout io.ReadCloser
|
|
||||||
stdout, err = command.StdoutPipe()
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
for true {
|
|
||||||
bytes := make([]byte, 1024)
|
|
||||||
_, err = stdout.Read(bytes)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
output := string(bytes)
|
|
||||||
|
|
||||||
compile := regexp.MustCompile(`[a-z\[\]\s]+([\d\.]+).[a-z\s]+([\d\.]+)([a-zA-Z]+)[a-z\s]+`)
|
|
||||||
if err != nil {
|
|
||||||
y.logger.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
res := compile.FindAllStringSubmatch(output, -1)
|
|
||||||
if len(res) != 0 && strings.Contains(res[0][0], "download") && len(res[0]) >= 2 {
|
|
||||||
progress := res[0][1]
|
|
||||||
|
|
||||||
y.logger.Debugw("progress",
|
|
||||||
"percentage", progress,
|
|
||||||
"link", link)
|
|
||||||
updateEvent(progress)
|
|
||||||
}
|
|
||||||
|
|
||||||
time.Sleep(time.Millisecond * y.checkFrequencyMs)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
err = command.Start()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
y.logger.Warn(err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = command.Wait()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var dir []fs.DirEntry
|
|
||||||
dir, err = os.ReadDir(baseDir)
|
|
||||||
if err != nil {
|
|
||||||
y.logger.Error("Could not read directory")
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, fileInfo := range dir {
|
|
||||||
oldPath := fmt.Sprintf("%s/%s", baseDir, fileInfo.Name())
|
|
||||||
newPath := fmt.Sprintf("%s/%s", y.outputDirectory, fileInfo.Name())
|
|
||||||
|
|
||||||
if err := files.MoveFile(oldPath, newPath); err != nil {
|
|
||||||
return err
|
|
||||||
} else {
|
|
||||||
y.logger.Infow("moved file",
|
|
||||||
"fileName", fileInfo.Name())
|
|
||||||
if err := os.Remove(baseDir); err != nil {
|
|
||||||
y.logger.Warn("could not cleanup",
|
|
||||||
"path", oldPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
10
api/internal/core/ports/downloadhandler/downloadhandler.go
Normal file
10
api/internal/core/ports/downloadhandler/downloadhandler.go
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
package downloadhandler
|
||||||
|
|
||||||
|
type OnDownloadEventHandler interface {
|
||||||
|
OnTickEvent(downloadId string, progress string)
|
||||||
|
}
|
||||||
|
|
||||||
|
type DownloadHandler interface {
|
||||||
|
OnProgress(eventHandler OnDownloadEventHandler)
|
||||||
|
Download(link string, outputDir string, downloadId string) error
|
||||||
|
}
|
98
api/internal/core/ports/downloadhandler/youtube_dl.go
Normal file
98
api/internal/core/ports/downloadhandler/youtube_dl.go
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
package downloadhandler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/ovh/configstore"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"log"
|
||||||
|
"os/exec"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type YoutubeDlDownloader struct {
|
||||||
|
progressUpdate bool
|
||||||
|
checkFrequencyMs time.Duration
|
||||||
|
logger *zap.SugaredLogger
|
||||||
|
onDownloadEventHandler OnDownloadEventHandler
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewYoutubeDlDownloader(logger *zap.SugaredLogger) DownloadHandler {
|
||||||
|
var checkFrequencyMs int64
|
||||||
|
var featureProgressUpdate bool
|
||||||
|
var err error
|
||||||
|
|
||||||
|
checkFrequencyMs, err = configstore.GetItemValueInt("download_frequency_update_ms")
|
||||||
|
featureProgressUpdate, err = configstore.GetItemValueBool("download_progress_update")
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = exec.Command("youtube-dl", "--version").Output()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Youtube download (youtube-dl) isn't installed on the device")
|
||||||
|
}
|
||||||
|
|
||||||
|
return &YoutubeDlDownloader{
|
||||||
|
checkFrequencyMs: time.Duration(checkFrequencyMs),
|
||||||
|
progressUpdate: featureProgressUpdate,
|
||||||
|
logger: logger,
|
||||||
|
onDownloadEventHandler: nil,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (y *YoutubeDlDownloader) OnProgress(eventHandler OnDownloadEventHandler) {
|
||||||
|
y.onDownloadEventHandler = eventHandler
|
||||||
|
}
|
||||||
|
|
||||||
|
func (y *YoutubeDlDownloader) Download(link string, outputDir string, downloadId string) error {
|
||||||
|
filePath := fmt.Sprintf("%s/%s", outputDir, "%(title)s-%(id)s.%(ext)s")
|
||||||
|
command := exec.Command("youtube-dl",
|
||||||
|
"-R 3",
|
||||||
|
"-o",
|
||||||
|
filePath,
|
||||||
|
link,
|
||||||
|
)
|
||||||
|
|
||||||
|
stdout, err := command.StdoutPipe()
|
||||||
|
|
||||||
|
if y.progressUpdate {
|
||||||
|
go func() {
|
||||||
|
for true {
|
||||||
|
bytes := make([]byte, 1024)
|
||||||
|
_, err = stdout.Read(bytes)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
output := string(bytes)
|
||||||
|
|
||||||
|
if err = y.getParameters(output, link, downloadId); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(time.Millisecond * y.checkFrequencyMs)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
return command.Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (y *YoutubeDlDownloader) getParameters(output string, link string, downloadId string) error {
|
||||||
|
compile := regexp.MustCompile(`[a-z\[\]\s]+([\d\.]+).[a-z\s]+([\d\.]+)([a-zA-Z]+)[a-z\s]+`)
|
||||||
|
|
||||||
|
res := compile.FindAllStringSubmatch(output, -1)
|
||||||
|
if len(res) != 0 && strings.Contains(res[0][0], "download") && len(res[0]) >= 2 {
|
||||||
|
progress := res[0][1]
|
||||||
|
|
||||||
|
y.logger.Debugw("progress",
|
||||||
|
"percentage", progress,
|
||||||
|
"link", link)
|
||||||
|
if y.onDownloadEventHandler != nil {
|
||||||
|
y.onDownloadEventHandler.OnTickEvent(downloadId, progress)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
98
api/internal/core/ports/downloadhandler/yt-dlp.go
Normal file
98
api/internal/core/ports/downloadhandler/yt-dlp.go
Normal file
@ -0,0 +1,98 @@
|
|||||||
|
package downloadhandler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/ovh/configstore"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"log"
|
||||||
|
"os/exec"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type YtDlpDownloader struct {
|
||||||
|
progressUpdate bool
|
||||||
|
checkFrequencyMs time.Duration
|
||||||
|
logger *zap.SugaredLogger
|
||||||
|
onDownloadEventHandler OnDownloadEventHandler
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewYtDlpDownloader(logger *zap.SugaredLogger) DownloadHandler {
|
||||||
|
var checkFrequencyMs int64
|
||||||
|
var featureProgressUpdate bool
|
||||||
|
var err error
|
||||||
|
|
||||||
|
checkFrequencyMs, err = configstore.GetItemValueInt("download_frequency_update_ms")
|
||||||
|
featureProgressUpdate, err = configstore.GetItemValueBool("download_progress_update")
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = exec.Command("yt-dlp", "--version").Output()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Youtube download (youtube-dl) isn't installed on the device")
|
||||||
|
}
|
||||||
|
|
||||||
|
return &YtDlpDownloader{
|
||||||
|
checkFrequencyMs: time.Duration(checkFrequencyMs),
|
||||||
|
progressUpdate: featureProgressUpdate,
|
||||||
|
logger: logger,
|
||||||
|
onDownloadEventHandler: nil,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (y *YtDlpDownloader) OnProgress(eventHandler OnDownloadEventHandler) {
|
||||||
|
y.onDownloadEventHandler = eventHandler
|
||||||
|
}
|
||||||
|
|
||||||
|
func (y *YtDlpDownloader) Download(link string, outputDir string, downloadId string) error {
|
||||||
|
filePath := fmt.Sprintf("%s/%s", outputDir, "%(title)s-%(id)s.%(ext)s")
|
||||||
|
command := exec.Command("yt-dlp",
|
||||||
|
"-R 3",
|
||||||
|
"-o",
|
||||||
|
filePath,
|
||||||
|
link,
|
||||||
|
)
|
||||||
|
|
||||||
|
stdout, err := command.StdoutPipe()
|
||||||
|
|
||||||
|
if y.progressUpdate {
|
||||||
|
go func() {
|
||||||
|
for true {
|
||||||
|
bytes := make([]byte, 1024)
|
||||||
|
_, err = stdout.Read(bytes)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
output := string(bytes)
|
||||||
|
|
||||||
|
if err = y.getParameters(output, link, downloadId); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(time.Millisecond * y.checkFrequencyMs)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
return command.Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (y *YtDlpDownloader) getParameters(output string, link string, downloadId string) error {
|
||||||
|
compile := regexp.MustCompile(`[a-z\[\]\s]+([\d\.]+).[a-z\s\~]+([\d\.]+)([a-zA-Z]+)[a-z\s]+`)
|
||||||
|
|
||||||
|
res := compile.FindAllStringSubmatch(output, -1)
|
||||||
|
if len(res) != 0 && strings.Contains(res[0][0], "download") && len(res[0]) >= 2 {
|
||||||
|
progress := res[0][1]
|
||||||
|
|
||||||
|
y.logger.Debugw("progress",
|
||||||
|
"percentage", progress,
|
||||||
|
"link", link)
|
||||||
|
if y.onDownloadEventHandler != nil {
|
||||||
|
y.onDownloadEventHandler.OnTickEvent(downloadId, progress)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
39
api/internal/core/ports/filehandler/mover/local/mover.go
Normal file
39
api/internal/core/ports/filehandler/mover/local/mover.go
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
package local
|
||||||
|
|
||||||
|
import (
|
||||||
|
"downloader/internal/core/ports/filehandler/mover"
|
||||||
|
"downloader/pkg/files"
|
||||||
|
"fmt"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Mover struct {
|
||||||
|
logger *zap.SugaredLogger
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(logger *zap.SugaredLogger) mover.Mover {
|
||||||
|
return &Mover{logger: logger}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *Mover) Move(sourceDirectory string, destinationDirectory string) error {
|
||||||
|
dir, err := os.ReadDir(sourceDirectory)
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Error("Could not read directory")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, fileInfo := range dir {
|
||||||
|
oldPath := fmt.Sprintf("%s/%s", sourceDirectory, fileInfo.Name())
|
||||||
|
newPath := fmt.Sprintf("%s/%s", destinationDirectory, fileInfo.Name())
|
||||||
|
|
||||||
|
if err := files.MoveFile(oldPath, newPath); err != nil {
|
||||||
|
return err
|
||||||
|
} else {
|
||||||
|
m.logger.Infow("moved file",
|
||||||
|
"fileName", fileInfo.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
5
api/internal/core/ports/filehandler/mover/mover.go
Normal file
5
api/internal/core/ports/filehandler/mover/mover.go
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
package mover
|
||||||
|
|
||||||
|
type Mover interface {
|
||||||
|
Move(sourceDirectory string, destinationDirectory string) error
|
||||||
|
}
|
@ -0,0 +1,36 @@
|
|||||||
|
package destinationhandler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"downloader/internal/core/ports/filehandler/mover"
|
||||||
|
"github.com/ovh/configstore"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DestinationHandler interface {
|
||||||
|
Prepare() error
|
||||||
|
Move(sourcePath string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type localDestinationHandler struct {
|
||||||
|
outputDirectory string
|
||||||
|
mover mover.Mover
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(mover mover.Mover) DestinationHandler {
|
||||||
|
outputDirectory, err := configstore.GetItemValue("download_output")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return &localDestinationHandler{
|
||||||
|
outputDirectory: outputDirectory,
|
||||||
|
mover: mover,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l localDestinationHandler) Prepare() error {
|
||||||
|
return os.MkdirAll(l.outputDirectory, os.ModePerm)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l localDestinationHandler) Move(sourcePath string) error {
|
||||||
|
return l.mover.Move(sourcePath, l.outputDirectory)
|
||||||
|
}
|
40
api/internal/core/ports/fileorchestrator/fileorchestrator.go
Normal file
40
api/internal/core/ports/fileorchestrator/fileorchestrator.go
Normal file
@ -0,0 +1,40 @@
|
|||||||
|
package fileorchestrator
|
||||||
|
|
||||||
|
import (
|
||||||
|
"downloader/internal/core/ports/fileorchestrator/destinationhandler"
|
||||||
|
"downloader/internal/core/ports/fileorchestrator/sourcehandler"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type FileOrchestrator struct {
|
||||||
|
logger *zap.SugaredLogger
|
||||||
|
sourceHandler sourcehandler.SourceHandler
|
||||||
|
destinationHandler destinationhandler.DestinationHandler
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(logger *zap.SugaredLogger, sourceHandler sourcehandler.SourceHandler, destinationHandler destinationhandler.DestinationHandler) *FileOrchestrator {
|
||||||
|
return &FileOrchestrator{
|
||||||
|
logger: logger,
|
||||||
|
sourceHandler: sourceHandler,
|
||||||
|
destinationHandler: destinationHandler,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *FileOrchestrator) Begin(link string) (string, error) {
|
||||||
|
basePath, err := o.sourceHandler.Prepare(link)
|
||||||
|
err = o.destinationHandler.Prepare()
|
||||||
|
|
||||||
|
return basePath, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *FileOrchestrator) CleanUp(sourceDirectory string) {
|
||||||
|
err := o.sourceHandler.CleanUp(sourceDirectory)
|
||||||
|
if err != nil {
|
||||||
|
o.logger.Errorw("could not cleanup",
|
||||||
|
"path", sourceDirectory)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *FileOrchestrator) Move(sourcePath string) error {
|
||||||
|
return o.destinationHandler.Move(sourcePath)
|
||||||
|
}
|
@ -0,0 +1,39 @@
|
|||||||
|
package sourcehandler
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"fmt"
|
||||||
|
"github.com/ovh/configstore"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
type SourceHandler interface {
|
||||||
|
Prepare(link string) (string, error)
|
||||||
|
CleanUp(sourcePath string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type localSourceHandler struct {
|
||||||
|
outputTmpDirectory string
|
||||||
|
}
|
||||||
|
|
||||||
|
func New() SourceHandler {
|
||||||
|
outputTmpDirectory, err := configstore.GetItemValue("download_tmp_output")
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return &localSourceHandler{
|
||||||
|
outputTmpDirectory: outputTmpDirectory,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l localSourceHandler) Prepare(link string) (string, error) {
|
||||||
|
path := fmt.Sprintf("%s/%x",
|
||||||
|
l.outputTmpDirectory,
|
||||||
|
sha256.Sum256([]byte(link)))
|
||||||
|
err := os.MkdirAll(path, os.ModePerm)
|
||||||
|
return path, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l localSourceHandler) CleanUp(sourcePath string) error {
|
||||||
|
return os.Remove(sourcePath)
|
||||||
|
}
|
@ -3,9 +3,8 @@ package _default
|
|||||||
import (
|
import (
|
||||||
"downloader/internal/core/entities"
|
"downloader/internal/core/entities"
|
||||||
"downloader/internal/core/ports/download_request"
|
"downloader/internal/core/ports/download_request"
|
||||||
"downloader/internal/core/ports/downloader"
|
|
||||||
"downloader/internal/core/services/download"
|
"downloader/internal/core/services/download"
|
||||||
"fmt"
|
"downloader/internal/core/services/downloader"
|
||||||
"go.uber.org/zap"
|
"go.uber.org/zap"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -26,10 +25,7 @@ func (l localBackgroundService) Run(download *entities.Download) error {
|
|||||||
download.Status = "started"
|
download.Status = "started"
|
||||||
_ = l.repository.Update(download)
|
_ = l.repository.Update(download)
|
||||||
|
|
||||||
err := l.downloader.Download(download.Link, func(progress string) {
|
err := l.downloader.Download(download.Link, download.ID)
|
||||||
download.Status = fmt.Sprintf("in-progress: %s", progress)
|
|
||||||
_ = l.repository.Update(download)
|
|
||||||
})
|
|
||||||
download.Status = "done"
|
download.Status = "done"
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -0,0 +1,32 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"downloader/internal/core/ports/download_request"
|
||||||
|
"downloader/internal/core/ports/downloadhandler"
|
||||||
|
"fmt"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type onDownloadEventHandler struct {
|
||||||
|
repository download_request.Repository
|
||||||
|
logger *zap.SugaredLogger
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(repository download_request.Repository, logger *zap.SugaredLogger) downloadhandler.OnDownloadEventHandler {
|
||||||
|
return &onDownloadEventHandler{
|
||||||
|
repository: repository,
|
||||||
|
logger: logger,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (o *onDownloadEventHandler) OnTickEvent(downloadId string, progress string) {
|
||||||
|
download, err := o.repository.GetById(downloadId)
|
||||||
|
if err != nil {
|
||||||
|
o.logger.Warnw("could not finish updating progress as not download id available",
|
||||||
|
"downloadId", downloadId,
|
||||||
|
"progress", progress)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
download.Status = fmt.Sprintf("in-progress: %s", progress)
|
||||||
|
_ = o.repository.Update(download)
|
||||||
|
}
|
54
api/internal/core/services/downloader/downloader.go
Normal file
54
api/internal/core/services/downloader/downloader.go
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
package downloader
|
||||||
|
|
||||||
|
import (
|
||||||
|
"downloader/internal/core/ports/downloadhandler"
|
||||||
|
"downloader/internal/core/ports/fileorchestrator"
|
||||||
|
"errors"
|
||||||
|
"go.uber.org/zap"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Downloader interface {
|
||||||
|
Download(link string, downloadId string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type downloader struct {
|
||||||
|
logger *zap.SugaredLogger
|
||||||
|
downloadHandler downloadhandler.DownloadHandler
|
||||||
|
fileOrchestrator *fileorchestrator.FileOrchestrator
|
||||||
|
ondownloadhandler downloadhandler.OnDownloadEventHandler
|
||||||
|
}
|
||||||
|
|
||||||
|
func New(
|
||||||
|
logger *zap.SugaredLogger,
|
||||||
|
orchestrator *fileorchestrator.FileOrchestrator,
|
||||||
|
downloadHandler downloadhandler.DownloadHandler,
|
||||||
|
downloadEventHandler downloadhandler.OnDownloadEventHandler,
|
||||||
|
) Downloader {
|
||||||
|
return &downloader{
|
||||||
|
logger: logger,
|
||||||
|
downloadHandler: downloadHandler,
|
||||||
|
fileOrchestrator: orchestrator,
|
||||||
|
ondownloadhandler: downloadEventHandler,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (y *downloader) Download(link string, downloadId string) error {
|
||||||
|
basePath, err := y.fileOrchestrator.Begin(link)
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("could not prepare for this link")
|
||||||
|
}
|
||||||
|
defer y.fileOrchestrator.CleanUp(basePath)
|
||||||
|
|
||||||
|
y.downloadHandler.OnProgress(y.ondownloadhandler)
|
||||||
|
err = y.downloadHandler.Download(link, basePath, downloadId)
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("could not download file")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = y.fileOrchestrator.Move(basePath)
|
||||||
|
if err != nil {
|
||||||
|
return errors.New("could not move to final destination")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user