store: keep an in-memory index of routes, support lookup by path

Signed-off-by: Andrea Luzzardi <aluzzardi@gmail.com>

cmd/dagger/cmd/down.go

@@ -23,10 +23,13 @@ var downCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		lg := logger.New()
 		ctx := lg.WithContext(cmd.Context())
-		store := dagger.DefaultStore()
+		store, err := dagger.DefaultStore()
+		if err != nil {
+			lg.Fatal().Err(err).Msg("failed to load store")
+		}
 
 		routeName := getRouteName(ctx)
-		route, err := store.LookupRoute(ctx, routeName, nil)
+		st, err := store.LookupRouteByName(ctx, routeName)
 		if err != nil {
 			lg.
 				Fatal().
@@ -34,6 +37,13 @@ var downCmd = &cobra.Command{
 				Str("routeName", routeName).
 				Msg("failed to lookup route")
 		}
+		route, err := dagger.NewRoute(st)
+		if err != nil {
+			lg.
+				Fatal().
+				Err(err).
+				Msg("failed to initialize route")
+		}
 
 		// TODO: Implement options: --no-cache
 		if err := route.Down(ctx, nil); err != nil {

cmd/dagger/cmd/list.go

@@ -23,7 +23,10 @@ var listCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		lg := logger.New()
 		ctx := lg.WithContext(cmd.Context())
-		store := dagger.DefaultStore()
+		store, err := dagger.DefaultStore()
+		if err != nil {
+			lg.Fatal().Err(err).Msg("failed to load store")
+		}
 
 		routes, err := store.ListRoutes(ctx)
 		if err != nil {
@@ -33,8 +36,8 @@ var listCmd = &cobra.Command{
 				Msg("cannot list routes")
 		}
 
-		for _, name := range routes {
+		for _, r := range routes {
-			fmt.Println(name)
+			fmt.Println(r.Name)
 		}
 	},
 }

cmd/dagger/cmd/new.go

@@ -22,23 +22,36 @@ var newCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		lg := logger.New()
 		ctx := lg.WithContext(cmd.Context())
-		store := dagger.DefaultStore()
+		store, err := dagger.DefaultStore()
+		if err != nil {
+			lg.Fatal().Err(err).Msg("failed to load store")
+		}
 
 		upRouteFlag := viper.GetBool("up")
 
-		routeName := getRouteName(ctx)
+		st := &dagger.RouteState{
+			Name: getRouteName(ctx),
+		}
 
 		// TODO: Implement options: --layout-*, --setup
-		route, err := store.CreateRoute(ctx, routeName, nil)
+		err = store.CreateRoute(ctx, st)
 		if err != nil {
 			lg.Fatal().Err(err).Msg("failed to create route")
 		}
 		lg.
 			Info().
-			Str("routeId", route.ID()).
+			Str("routeId", st.ID).
-			Str("routeName", routeName).
+			Str("routeName", st.Name).
 			Msg("route created")
 
+		route, err := dagger.NewRoute(st)
+		if err != nil {
+			lg.
+				Fatal().
+				Err(err).
+				Msg("failed to initialize route")
+		}
+
 		if upRouteFlag {
 			routeUp(ctx, route)
 		}

cmd/dagger/cmd/query.go

@@ -24,10 +24,13 @@ var queryCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		lg := logger.New()
 		ctx := lg.WithContext(cmd.Context())
-		store := dagger.DefaultStore()
+		store, err := dagger.DefaultStore()
+		if err != nil {
+			lg.Fatal().Err(err).Msg("failed to load store")
+		}
 
 		routeName := getRouteName(ctx)
-		route, err := store.LookupRoute(ctx, routeName, nil)
+		st, err := store.LookupRouteByName(ctx, routeName)
 		if err != nil {
 			lg.
 				Fatal().
@@ -36,6 +39,14 @@ var queryCmd = &cobra.Command{
 				Msg("failed to lookup route")
 		}
 
+		route, err := dagger.NewRoute(st)
+		if err != nil {
+			lg.
+				Fatal().
+				Err(err).
+				Msg("failed to initialize route")
+		}
+
 		expr := args[0]
 
 		out, err := route.Query(ctx, expr, nil)

cmd/dagger/cmd/up.go

@@ -23,10 +23,13 @@ var upCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		lg := logger.New()
 		ctx := lg.WithContext(cmd.Context())
-		store := dagger.DefaultStore()
+		store, err := dagger.DefaultStore()
+		if err != nil {
+			lg.Fatal().Err(err).Msg("failed to load store")
+		}
 
 		routeName := getRouteName(ctx)
-		route, err := store.LookupRoute(ctx, routeName, nil)
+		st, err := store.LookupRouteByName(ctx, routeName)
 		if err != nil {
 			lg.
 				Fatal().
@@ -35,6 +38,14 @@ var upCmd = &cobra.Command{
 				Msg("failed to lookup route")
 		}
 
+		route, err := dagger.NewRoute(st)
+		if err != nil {
+			lg.
+				Fatal().
+				Err(err).
+				Msg("failed to initialize route")
+		}
+
 		// TODO: Implement options: --no-cache
 		routeUp(ctx, route)
 	},

dagger/route.go

@@ -49,7 +49,16 @@ func (r *RouteState) AddInput(key string, value Input) error {
 // For example RemoveInputs("foo.bar") will remove all inputs
 //   at foo.bar, foo.bar.baz, etc.
 func (r *RouteState) RemoveInputs(key string) error {
-	panic("NOT IMPLEMENTED")
+	newInputs := make([]inputKV, 0, len(r.Inputs))
+	for _, i := range r.Inputs {
+		if i.Key == key {
+			continue
+		}
+		newInputs = append(newInputs, i)
+	}
+	r.Inputs = newInputs
+
+	return nil
 }
 
 type Route struct {

dagger/store.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"os"
 	"path"
+	"sync"
 
 	"github.com/google/uuid"
 )
@@ -16,107 +17,205 @@ const (
 
 type Store struct {
 	root string
+
+	l sync.RWMutex
+
+	routes map[string]*RouteState
+
+	// Various indices for fast lookups
+	routesByName map[string]*RouteState
+	routesByPath map[string]*RouteState
+	pathsByRoute map[string][]string
 }
 
-func NewStore(root string) *Store {
+func NewStore(root string) (*Store, error) {
-	return &Store{
+	store := &Store{
-		root: root,
+		root:         root,
+		routes:       make(map[string]*RouteState),
+		routesByName: make(map[string]*RouteState),
+		routesByPath: make(map[string]*RouteState),
+		pathsByRoute: make(map[string][]string),
 	}
+	return store, store.loadAll()
 }
 
-func DefaultStore() *Store {
+func DefaultStore() (*Store, error) {
 	return NewStore(os.ExpandEnv(defaultStoreRoot))
 }
 
-type CreateOpts struct{}
-
-func (s *Store) CreateRoute(ctx context.Context, name string, o *CreateOpts) (*Route, error) {
-	r, err := s.LookupRoute(ctx, name, &LookupOpts{})
-	if err != nil && !errors.Is(err, os.ErrNotExist) {
-		return nil, err
-	}
-	if r != nil {
-		return nil, os.ErrExist
-	}
-	r, err = NewRoute(
-		&RouteState{
-			ID:   uuid.New().String(),
-			Name: name,
-		},
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return r, s.syncRoute(r)
-}
-
-type UpdateOpts struct{}
-
-func (s *Store) UpdateRoute(ctx context.Context, r *Route, o *UpdateOpts) error {
-	return s.syncRoute(r)
-}
-
-type DeleteOpts struct{}
-
-func (s *Store) DeleteRoute(ctx context.Context, r *Route, o *DeleteOpts) error {
-	return os.Remove(s.routePath(r.st.Name))
-}
-
-type LookupOpts struct{}
-
-func (s *Store) LookupRoute(ctx context.Context, name string, o *LookupOpts) (*Route, error) {
-	data, err := os.ReadFile(s.routePath(name))
-	if err != nil {
-		return nil, err
-	}
-	var st RouteState
-	if err := json.Unmarshal(data, &st); err != nil {
-		return nil, err
-	}
-	return &Route{
-		st: &st,
-	}, nil
-}
-
-type LoadOpts struct{}
-
-func (s *Store) LoadRoute(ctx context.Context, id string, o *LoadOpts) (*Route, error) {
-	panic("NOT IMPLEMENTED")
-}
-
-func (s *Store) ListRoutes(ctx context.Context) ([]string, error) {
-	routes := []string{}
-
-	files, err := os.ReadDir(s.root)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, f := range files {
-		if f.IsDir() {
-			routes = append(routes, f.Name())
-		}
-	}
-
-	return routes, nil
-}
-
 func (s *Store) routePath(name string) string {
 	return path.Join(s.root, name, "route.json")
 }
 
-func (s *Store) syncRoute(r *Route) error {
+func (s *Store) loadAll() error {
-	p := s.routePath(r.st.Name)
+	files, err := os.ReadDir(s.root)
+	if err != nil {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil
+		}
+		return err
+	}
+
+	for _, f := range files {
+		if !f.IsDir() {
+			continue
+		}
+		if err := s.loadRoute(f.Name()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *Store) loadRoute(name string) error {
+	data, err := os.ReadFile(s.routePath(name))
+	if err != nil {
+		return err
+	}
+	var st RouteState
+	if err := json.Unmarshal(data, &st); err != nil {
+		return err
+	}
+	s.indexRoute(&st)
+	return nil
+}
+
+func (s *Store) syncRoute(r *RouteState) error {
+	p := s.routePath(r.Name)
 
 	if err := os.MkdirAll(path.Dir(p), 0755); err != nil {
 		return err
 	}
 
-	data, err := json.MarshalIndent(r.st, "", "  ")
+	data, err := json.MarshalIndent(r, "", "  ")
 	if err != nil {
 		return err
 	}
 
-	return os.WriteFile(p, data, 0644)
+	if err := os.WriteFile(p, data, 0644); err != nil {
+		return err
+	}
+
+	s.reindexRoute(r)
+
+	return nil
+}
+
+func (s *Store) indexRoute(r *RouteState) {
+	s.routes[r.ID] = r
+	s.routesByName[r.Name] = r
+
+	mapPath := func(i Input) {
+		d, ok := i.(*dirInput)
+		if !ok {
+			return
+		}
+		s.routesByPath[d.Path] = r
+		s.pathsByRoute[r.ID] = append(s.pathsByRoute[r.ID], d.Path)
+	}
+
+	mapPath(r.LayoutSource)
+	for _, i := range r.Inputs {
+		mapPath(i.Value)
+	}
+}
+
+func (s *Store) deindexRoute(id string) {
+	r, ok := s.routes[id]
+	if !ok {
+		return
+	}
+	delete(s.routes, r.ID)
+	delete(s.routesByName, r.Name)
+
+	for _, p := range s.pathsByRoute[r.ID] {
+		delete(s.routesByPath, p)
+	}
+	delete(s.pathsByRoute, r.ID)
+}
+
+func (s *Store) reindexRoute(r *RouteState) {
+	s.deindexRoute(r.ID)
+	s.indexRoute(r)
+}
+
+func (s *Store) CreateRoute(ctx context.Context, st *RouteState) error {
+	s.l.Lock()
+	defer s.l.Unlock()
+
+	if _, ok := s.routesByName[st.Name]; ok {
+		return os.ErrExist
+	}
+
+	st.ID = uuid.New().String()
+	return s.syncRoute(st)
+}
+
+type UpdateOpts struct{}
+
+func (s *Store) UpdateRoute(ctx context.Context, r *RouteState, o *UpdateOpts) error {
+	s.l.Lock()
+	defer s.l.Unlock()
+
+	return s.syncRoute(r)
+}
+
+type DeleteOpts struct{}
+
+func (s *Store) DeleteRoute(ctx context.Context, r *RouteState, o *DeleteOpts) error {
+	s.l.Lock()
+	defer s.l.Unlock()
+
+	if err := os.Remove(s.routePath(r.Name)); err != nil {
+		return err
+	}
+	s.deindexRoute(r.ID)
+	return nil
+}
+
+func (s *Store) LookupRouteByID(ctx context.Context, id string) (*RouteState, error) {
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	st, ok := s.routes[id]
+	if !ok {
+		return nil, os.ErrNotExist
+	}
+	return st, nil
+}
+
+func (s *Store) LookupRouteByName(ctx context.Context, name string) (*RouteState, error) {
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	st, ok := s.routesByName[name]
+	if !ok {
+		return nil, os.ErrNotExist
+	}
+	return st, nil
+}
+
+func (s *Store) LookupRouteByPath(ctx context.Context, path string) (*RouteState, error) {
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	st, ok := s.routesByPath[path]
+	if !ok {
+		return nil, os.ErrNotExist
+	}
+	return st, nil
+}
+
+func (s *Store) ListRoutes(ctx context.Context) ([]*RouteState, error) {
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	routes := make([]*RouteState, 0, len(s.routes))
+
+	for _, st := range s.routes {
+		routes = append(routes, st)
+	}
+
+	return routes, nil
 }

dagger/store_test.go

@@ -9,29 +9,91 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestStore(t *testing.T) {
+func TestStoreLoad(t *testing.T) {
 	ctx := context.TODO()
 
 	root, err := os.MkdirTemp(os.TempDir(), "dagger-*")
 	require.NoError(t, err)
-	store := NewStore(root)
+	store, err := NewStore(root)
+	require.NoError(t, err)
 
-	_, err = store.LookupRoute(ctx, "notexist", nil)
+	_, err = store.LookupRouteByName(ctx, "notexist")
 	require.Error(t, err)
 	require.True(t, errors.Is(err, os.ErrNotExist))
 
-	r, err := store.CreateRoute(ctx, "test", nil)
+	st := &RouteState{
-	require.NoError(t, err)
+		Name: "test",
-	require.NotNil(t, r)
+	}
-	require.Equal(t, "test", r.Name())
+	require.NoError(t, store.CreateRoute(ctx, st))
 
-	r, err = store.LookupRoute(ctx, "test", nil)
+	checkRoutes := func(store *Store) {
-	require.NoError(t, err)
+		r, err := store.LookupRouteByID(ctx, st.ID)
-	require.NotNil(t, r)
+		require.NoError(t, err)
-	require.Equal(t, "test", r.Name())
+		require.NotNil(t, r)
+		require.Equal(t, "test", r.Name)
 
-	routes, err := store.ListRoutes(ctx)
+		r, err = store.LookupRouteByName(ctx, "test")
+		require.NoError(t, err)
+		require.NotNil(t, r)
+		require.Equal(t, "test", r.Name)
+
+		routes, err := store.ListRoutes(ctx)
+		require.NoError(t, err)
+		require.Len(t, routes, 1)
+		require.Equal(t, "test", routes[0].Name)
+	}
+
+	checkRoutes(store)
+
+	// Reload the routes from disk and check again
+	newStore, err := NewStore(root)
+	require.NoError(t, err)
+	checkRoutes(newStore)
+}
+
+func TestStoreLookupByPath(t *testing.T) {
+	ctx := context.TODO()
+
+	root, err := os.MkdirTemp(os.TempDir(), "dagger-*")
+	require.NoError(t, err)
+	store, err := NewStore(root)
+	require.NoError(t, err)
+
+	st := &RouteState{
+		Name: "test",
+	}
+	require.NoError(t, st.AddInput("foo", DirInput("/test/path", []string{})))
+	require.NoError(t, store.CreateRoute(ctx, st))
+
+	// Lookup by path
+	r, err := store.LookupRouteByPath(ctx, "/test/path")
+	require.NoError(t, err)
+	require.NotNil(t, r)
+	require.Equal(t, st.ID, r.ID)
+
+	// Add a new path
+	require.NoError(t, st.AddInput("bar", DirInput("/test/anotherpath", []string{})))
+	require.NoError(t, store.UpdateRoute(ctx, st, nil))
+
+	// Lookup by the previous path
+	r, err = store.LookupRouteByPath(ctx, "/test/path")
+	require.NoError(t, err)
+	require.Equal(t, st.ID, r.ID)
+
+	// Lookup by the new path
+	r, err = store.LookupRouteByPath(ctx, "/test/anotherpath")
+	require.NoError(t, err)
+	require.Equal(t, st.ID, r.ID)
+
+	// Remove a path
+	require.NoError(t, st.RemoveInputs("foo"))
+	require.NoError(t, store.UpdateRoute(ctx, st, nil))
+
+	// Lookup by the removed path should fail
+	_, err = store.LookupRouteByPath(ctx, "/test/path")
+	require.Error(t, err)
+
+	// Lookup by the other path should still work
+	_, err = store.LookupRouteByPath(ctx, "/test/anotherpath")
 	require.NoError(t, err)
-	require.Len(t, routes, 1)
-	require.Equal(t, "test", routes[0])
 }