add: knowledge delete-file & knowledge get-file

pkg/client/client.go

@@ -29,6 +29,8 @@ type Client interface {
 	CreateDataset(ctx context.Context, datasetID string) (*index.Dataset, error)
 	DeleteDataset(ctx context.Context, datasetID string) error
 	GetDataset(ctx context.Context, datasetID string) (*index.Dataset, error)
+	FindFile(ctx context.Context, searchFile index.File) (*index.File, error)
+	DeleteFile(ctx context.Context, datasetID, fileID string) error
 	ListDatasets(ctx context.Context) ([]types.Dataset, error)
 	Ingest(ctx context.Context, datasetID string, name string, data []byte, opts datastore.IngestOpts) ([]string, error)
 	IngestPaths(ctx context.Context, datasetID string, opts *IngestPathsOpts, paths ...string) (int, error) // returns number of files ingested

pkg/client/default.go

@@ -30,6 +30,16 @@ func NewDefaultClient(serverURL string) *DefaultClient {
 	}
 }
 
+func (c *DefaultClient) FindFile(_ context.Context, searchFile index.File) (*index.File, error) {
+	// TODO: implement
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (c *DefaultClient) DeleteFile(_ context.Context, datasetID, fileID string) error {
+	// TODO: implement
+	return fmt.Errorf("not implemented")
+}
+
 func (c *DefaultClient) CreateDataset(_ context.Context, datasetID string) (*index.Dataset, error) {
 	ds := types.Dataset{
 		ID: datasetID,

pkg/client/standalone.go

@@ -25,6 +25,14 @@ func NewStandaloneClient(ds *datastore.Datastore) (*StandaloneClient, error) {
 	}, nil
 }
 
+func (c *StandaloneClient) FindFile(ctx context.Context, searchFile index.File) (*index.File, error) {
+	return c.Datastore.FindFile(ctx, searchFile)
+}
+
+func (c *StandaloneClient) DeleteFile(ctx context.Context, datasetID, fileID string) error {
+	return c.Datastore.DeleteFile(ctx, datasetID, fileID)
+}
+
 func (c *StandaloneClient) CreateDataset(ctx context.Context, datasetID string) (*index.Dataset, error) {
 	ds := index.Dataset{
 		ID: datasetID,

pkg/cmd/delete_file.go

@@ -0,0 +1,75 @@
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/gptscript-ai/knowledge/pkg/datastore"
+	"github.com/gptscript-ai/knowledge/pkg/index"
+	"github.com/spf13/cobra"
+)
+
+type ClientDeleteFile struct {
+	Client
+	Dataset string `usage:"Target Dataset ID" short:"d" default:"default"`
+}
+
+func (s *ClientDeleteFile) Customize(cmd *cobra.Command) {
+	cmd.Use = "delete-file <file-id|file-abs-path>"
+	cmd.Short = "Delete a file from a dataset"
+	cmd.Args = cobra.ExactArgs(1)
+}
+
+func (s *ClientDeleteFile) Run(cmd *cobra.Command, args []string) error {
+	c, err := s.getClient(cmd.Context())
+	if err != nil {
+		return err
+	}
+
+	fileRef := args[0]
+
+	searchFile := index.File{
+		Dataset: s.Dataset,
+	}
+
+	if strings.HasPrefix(fileRef, "/") {
+		searchFile.AbsolutePath = fileRef
+	} else if _, err := uuid.Parse(fileRef); err == nil {
+		searchFile.ID = fileRef
+	} else {
+		finfo, err := os.Stat(fileRef)
+		if err != nil {
+			return fmt.Errorf("fileref is not a valid filepath or UUID - failed to stat relative path: %w", err)
+		}
+		if finfo.IsDir() {
+			return fmt.Errorf("fileref is a directory, not a file")
+		}
+		searchFile.AbsolutePath, err = filepath.Abs(fileRef)
+		if err != nil {
+			return fmt.Errorf("failed to get absolute path: %w", err)
+		}
+	}
+
+	file, err := c.FindFile(cmd.Context(), searchFile)
+	if errors.Is(err, datastore.ErrDBFileNotFound) || file == nil {
+		slog.Info("File not found", "file", searchFile)
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	err = c.DeleteFile(cmd.Context(), s.Dataset, file.ID)
+	if err != nil {
+		return fmt.Errorf("failed to delete file: %w", err)
+	}
+
+	fmt.Printf("File %s (%s) deleted\n", file.ID, file.AbsolutePath)
+
+	return nil
+}

pkg/cmd/get_file.go

@@ -0,0 +1,75 @@
+package cmd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/gptscript-ai/knowledge/pkg/datastore"
+	"github.com/gptscript-ai/knowledge/pkg/index"
+	"github.com/spf13/cobra"
+)
+
+type ClientGetFile struct {
+	Client
+	Dataset string `usage:"Target Dataset ID" short:"d" default:"default"`
+}
+
+func (s *ClientGetFile) Customize(cmd *cobra.Command) {
+	cmd.Use = "get-file <file-id|file-abs-path>"
+	cmd.Short = "Get a file from a dataset"
+	cmd.Args = cobra.ExactArgs(1)
+}
+
+func (s *ClientGetFile) Run(cmd *cobra.Command, args []string) error {
+	c, err := s.getClient(cmd.Context())
+	if err != nil {
+		return err
+	}
+
+	fileRef := args[0]
+
+	searchFile := index.File{
+		Dataset: s.Dataset,
+	}
+
+	if strings.HasPrefix(fileRef, "/") {
+		searchFile.AbsolutePath = fileRef
+	} else if _, err := uuid.Parse(fileRef); err == nil {
+		searchFile.ID = fileRef
+	} else {
+		finfo, err := os.Stat(fileRef)
+		if err != nil {
+			return fmt.Errorf("fileref is not a valid filepath or UUID - failed to stat relative path: %w", err)
+		}
+		if finfo.IsDir() {
+			return fmt.Errorf("fileref is a directory, not a file")
+		}
+		searchFile.AbsolutePath, err = filepath.Abs(fileRef)
+		if err != nil {
+			return fmt.Errorf("failed to get absolute path: %w", err)
+		}
+	}
+
+	file, err := c.FindFile(cmd.Context(), searchFile)
+	if err != nil {
+		if errors.Is(err, datastore.ErrDBFileNotFound) {
+			fmt.Printf("File not found: %s\n", fileRef)
+			return nil
+		}
+		return err
+	}
+
+	jsonOutput, err := json.Marshal(file)
+	if err != nil {
+		return fmt.Errorf("failed to marshal file: %w", err)
+	}
+
+	fmt.Println(string(jsonOutput))
+
+	return c.DeleteFile(cmd.Context(), s.Dataset, file.ID)
+}

pkg/cmd/main.go

@@ -25,6 +25,8 @@ func New() *cobra.Command {
 		new(ClientListDatasets),
 		new(ClientIngest),
 		new(ClientDeleteDataset),
+		new(ClientDeleteFile),
+		new(ClientGetFile),
 		new(ClientRetrieve),
 		new(ClientResetDatastore),
 		new(ClientAskDir),

pkg/datastore/files.go

@@ -7,6 +7,7 @@ import (
 	"log/slog"
 
 	"github.com/gptscript-ai/knowledge/pkg/index"
+	"gorm.io/gorm"
 )
 
 // ErrDBFileNotFound is returned when a file is not found.
@@ -62,3 +63,24 @@ func (s *Datastore) PruneFiles(ctx context.Context, datasetID string, pathPrefix
 
 	return files, nil
 }
+
+func (s *Datastore) FindFile(ctx context.Context, searchFile index.File) (*index.File, error) {
+	if searchFile.Dataset == "" {
+		return nil, fmt.Errorf("dataset must be provided")
+	}
+
+	var file index.File
+	var tx *gorm.DB
+	if searchFile.ID != "" {
+		tx = s.Index.WithContext(ctx).Preload("Documents").Where("dataset = ? AND id = ?", searchFile.Dataset, searchFile.ID).First(&file)
+	} else if searchFile.AbsolutePath != "" {
+		tx = s.Index.WithContext(ctx).Preload("Documents").Where("dataset = ? AND absolute_path = ?", searchFile.Dataset, searchFile.AbsolutePath).First(&file)
+	} else {
+		return nil, fmt.Errorf("either fileID or fileAbsPath must be provided")
+	}
+	if tx.Error != nil {
+		return nil, ErrDBFileNotFound
+	}
+
+	return &file, nil
+}