Add base site, needs clean-up

api/go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-chi/chi v1.5.4 // indirect
+	github.com/go-chi/cors v1.2.0 // indirect
 	github.com/go-chi/render v1.0.1 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect

api/go.sum

@@ -7,6 +7,8 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs=
 github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg=
+github.com/go-chi/cors v1.2.0 h1:tV1g1XENQ8ku4Bq3K9ub2AtgG+p16SmzeMSGTwrOKdE=
+github.com/go-chi/cors v1.2.0/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
 github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8=
 github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=

api/internal/app/api/download/request_download.go

@@ -1,6 +1,7 @@
 package download
 
 import (
+	"context"
 	"downloader/internal/app/api/common/responses"
 	"downloader/internal/core/entities"
 	"errors"
@@ -48,6 +49,12 @@ func (a *api) requestDownload(w http.ResponseWriter, r *http.Request) {
 
 func (a *api) getDownloads(writer http.ResponseWriter, request *http.Request) {
 	ctx := request.Context()
+
+	if done := request.URL.Query().Get("done"); done == "true" {
+		_ = a.getDoneDownloads(writer, request, ctx)
+		return
+	}
+
 	active := request.URL.Query().Get("active") == "true"
 	downloads, err := a.drService.GetAll(ctx, active)
 	if err != nil {
@@ -59,7 +66,20 @@ func (a *api) getDownloads(writer http.ResponseWriter, request *http.Request) {
 		_ = render.Render(writer, request, responses.ErrInvalidRequest(err))
 		return
 	}
+}
 
+func (a *api) getDoneDownloads(writer http.ResponseWriter, request *http.Request, ctx context.Context) bool {
+	downloads, err := a.drService.GetDone(ctx)
+	if err != nil {
+		_ = render.Render(writer, request, responses.ErrInvalidRequest(err))
+		return true
+	}
+
+	if err = render.RenderList(writer, request, newDownloadsResponse(downloads)); err != nil {
+		_ = render.Render(writer, request, responses.ErrInvalidRequest(err))
+		return true
+	}
+	return false
 }
 
 func (a *api) getDownloadById(w http.ResponseWriter, r *http.Request) {

api/internal/app/infrastructure/logger/logger.go

@@ -10,13 +10,13 @@ func New() *zap.SugaredLogger {
 	consoleEncoder := zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())
 	jsonEncoder := zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig())
 
-	consoleDebugging := zapcore.Lock(os.Stdout)
 	lowPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
 		return lvl < zapcore.ErrorLevel
 	})
 	highPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool {
 		return lvl >= zapcore.ErrorLevel
 	})
+	consoleDebugging := zapcore.Lock(os.Stdout)
 	consoleErrors := zapcore.Lock(os.Stderr)
 	core := zapcore.NewTee(
 		zapcore.NewCore(jsonEncoder, consoleErrors, highPriority),

api/internal/app/router/router.go

@@ -17,6 +17,7 @@ import (
 	"downloader/pkg/common/uuid"
 	"github.com/go-chi/chi"
 	"github.com/go-chi/chi/middleware"
+	"github.com/go-chi/cors"
 	"net/http"
 	"time"
 )
@@ -49,7 +50,14 @@ func (router *router) setupMiddleware() *router {
 	router.internalRouter.Use(middleware.RealIP)
 	router.internalRouter.Use(middleware.Recoverer)
 	router.internalRouter.Use(middleware.Timeout(time.Second * 60))
-
+	router.internalRouter.Use(cors.Handler(cors.Options{
+		AllowedOrigins:   []string{"https://*", "http://*"},
+		AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
+		AllowedHeaders:   []string{"Accept", "Authorization", "Content-Type", "X-CSRF-Token"},
+		ExposedHeaders:   []string{"Link"},
+		AllowCredentials: false,
+		MaxAge:           300,
+	}))
 	return router
 }
 
@@ -80,6 +88,6 @@ func setupDownloadRoute(router *router) {
 	downloadApi := download.New(drService)
 	downloadApi.SetupDownloadApi(router.internalRouter)
 
-	cleanupJob := cleanup.New(drRepository, newLogger)
+	cleanupJob := cleanup.New(drRepository, newLogger, fileOrchestrator)
 	cleanupJob.RunOnSchedule()
 }

api/internal/core/ports/download_request/repository.go

@@ -12,4 +12,5 @@ type Repository interface {
 	Get(ctx context.Context, active bool) ([]*entities.Download, error)
 	GetOldOrStuck(ctx context.Context) ([]*entities.Download, error)
 	BatchDelete(ctx context.Context, requests []*entities.Download) error
+	GetDone(ctx context.Context) ([]*entities.Download, error)
 }

api/internal/core/ports/download_request/sql/repository.go

@@ -80,13 +80,22 @@ func (r repository) Update(ctx context.Context, download *entities.Download) err
 
 func (r repository) Get(ctx context.Context, active bool) ([]*entities.Download, error) {
 	var downloads []Download
-	err := r.db.NewSelect().
+	query := r.db.NewSelect().
 		Model(&downloads).
-		Column("id", "status", "link").
-		Where("status LIKE ?", "in-progress%").
-		Limit(20).
-		Order("created_at ASC").
-		Scan(ctx)
+		Column("id", "status", "link")
+
+	var err error
+	if active {
+		err = query.
+			Where("status LIKE ? OR status = ?", "in-progress%", "scheduled").
+			Order("created_at ASC").
+			Scan(ctx)
+	} else {
+		err = query.
+			Order("created_at ASC").
+			Scan(ctx)
+	}
+
 	if err != nil {
 		return nil, err
 	}
@@ -109,7 +118,7 @@ func (r repository) GetOldOrStuck(ctx context.Context) ([]*entities.Download, er
 	err := r.db.NewSelect().
 		Model(&downloads).
 		Column("id", "status", "link").
-		Where("status LIKE ?", "in-progress%").
+		Where("status LIKE ? OR status = ?", "in-progress%", "scheduled").
 		Where("updated_at < now() - interval '1 minutes'").
 		Limit(20).
 		Order("created_at ASC").
@@ -152,3 +161,28 @@ func (r repository) BatchDelete(ctx context.Context, requests []*entities.Downlo
 
 	return nil
 }
+
+func (r repository) GetDone(ctx context.Context) ([]*entities.Download, error) {
+	var downloads []Download
+	err := r.db.NewSelect().
+		Model(&downloads).
+		Column("id", "status", "link").
+		Where("status = ?", "done").
+		Order("created_at ASC").
+		Scan(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var responseDownloads []*entities.Download
+	for _, download := range downloads {
+		responseDownloads = append(responseDownloads, &entities.Download{
+			ID:     download.ID,
+			Status: download.Status,
+			Link:   download.Link,
+		})
+	}
+
+	return responseDownloads, nil
+}

api/internal/core/ports/fileorchestrator/sourcehandler/sourcehandler.go

@@ -35,5 +35,8 @@ func (l localSourceHandler) Prepare(link string) (string, error) {
 }
 
 func (l localSourceHandler) CleanUp(sourcePath string) error {
-	return os.Remove(sourcePath)
+	if _, err := os.Stat(sourcePath); os.IsNotExist(err) {
+		return nil
+	}
+	return os.RemoveAll(sourcePath)
 }

api/internal/core/services/cleanup/cleanup.go

@@ -3,6 +3,7 @@ package cleanup
 import (
 	"context"
 	"downloader/internal/core/ports/download_request"
+	"downloader/internal/core/ports/fileorchestrator"
 	"go.uber.org/zap"
 	"time"
 )
@@ -10,10 +11,11 @@ import (
 type cleanUp struct {
 	repository download_request.Repository
 	logger     *zap.SugaredLogger
+	fo         *fileorchestrator.FileOrchestrator
 }
 
-func New(repository download_request.Repository, logger *zap.SugaredLogger) *cleanUp {
-	return &cleanUp{repository: repository, logger: logger}
+func New(repository download_request.Repository, logger *zap.SugaredLogger, fo *fileorchestrator.FileOrchestrator) *cleanUp {
+	return &cleanUp{repository: repository, logger: logger, fo: fo}
 }
 
 func (c *cleanUp) RunOnSchedule() {
@@ -21,14 +23,27 @@ func (c *cleanUp) RunOnSchedule() {
 	go func() {
 		for true {
 			requests, err := c.repository.GetOldOrStuck(ctx)
-			if err == nil {
-				c.logger.Debugw("Cleaning up downloads",
-					"downloads", requests)
-				_ = c.repository.BatchDelete(ctx, requests)
-			} else {
+			if err != nil {
 				c.logger.Warn("could not process old or stuck in-progress jobs")
+				time.Sleep(5 * time.Minute)
+				continue
 			}
 
+			c.logger.Debugw("Cleaning up downloads",
+				"downloads", requests)
+
+			for _, request := range requests {
+				basePath, err := c.fo.Begin(request.Link)
+				if err != nil {
+					c.logger.Warnw("could not process request",
+						"downloadId", request.ID)
+					continue
+				}
+
+				c.fo.CleanUp(basePath)
+			}
+			_ = c.repository.BatchDelete(ctx, requests)
+
 			time.Sleep(time.Minute)
 		}
 	}()

api/internal/core/services/download/default/service.go

@@ -56,3 +56,7 @@ func (l *localService) Get(ctx context.Context, id string) (*entities.Download,
 func (l *localService) GetAll(ctx context.Context, active bool) ([]*entities.Download, error) {
 	return l.repository.Get(ctx, active)
 }
+
+func (l *localService) GetDone(ctx context.Context) ([]*entities.Download, error) {
+	return l.repository.GetDone(ctx)
+}

api/internal/core/services/download/service.go

@@ -9,4 +9,5 @@ type Service interface {
 	Schedule(ctx context.Context, link string) (*entities.Download, error)
 	Get(ctx context.Context, id string) (*entities.Download, error)
 	GetAll(ctx context.Context, active bool) ([]*entities.Download, error)
+	GetDone(ctx context.Context) ([]*entities.Download, error)
 }