Files' digest is calculated when they're added

Author: ItalyPaleAle

crypto/metadata.pb.go

@@ -16,7 +16,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
 
 /*
-//Build with:
+Build with:
 
 ```sh
 protoc \
@@ -38,7 +38,10 @@ option go_package = ".;crypto";
 
 // Metadata message
 message Metadata {
+    // File's original name
     string name = 1 [json_name="n"];
+    // File's content type
     string content_type = 2 [json_name="ct"];
+    // File size
     int64 size = 3 [json_name="sz"];
 }

crypto/metadata.proto

@@ -39,6 +39,8 @@ type FolderList struct {
 	FileId    string     `json:"fileId,omitempty"`
 	Date      *time.Time `json:"date,omitempty"`
 	MimeType  string     `json:"mimeType,omitempty"`
+	Digest    []byte     `json:"digest,omitempty"`
+	Size      int64      `json:"size,omitempty"`
 }
 
 // IndexStats contains the result of the
@@ -173,7 +175,7 @@ func (i *Index) save(obj *pb.IndexFile) error {
 }
 
 // AddFile adds a file to the index
-func (i *Index) AddFile(path string, fileId []byte, mimeType string) error {
+func (i *Index) AddFile(path string, fileId []byte, mimeType string, size int64, digest []byte) error {
 	// path must be at least 2 characters (with / being one)
 	if len(path) < 2 {
 		return errors.New("path name is too short")
@@ -186,6 +188,14 @@ func (i *Index) AddFile(path string, fileId []byte, mimeType string) error {
 	if strings.HasSuffix(path, "/") {
 		return errors.New("path must not end with /")
 	}
+	// File size must not be negative (but can be empty)
+	if size < 0 {
+		return errors.New("invalid file size")
+	}
+	// If the digest is empty, ensure it's null
+	if len(digest) < 1 {
+		digest = nil
+	}
 
 	// Force a refresh of the index
 	if err := i.Refresh(true); err != nil {
@@ -210,6 +220,8 @@ func (i *Index) AddFile(path string, fileId []byte, mimeType string) error {
 			Seconds: time.Now().Unix(),
 		},
 		MimeType: mimeType,
+		Size:     size,
+		Digest:   digest,
 	}
 	elements := append(i.cache.Elements, fileEl)
 
@@ -317,6 +329,8 @@ func (i *Index) GetFileById(fileId string) (*FolderList, error) {
 		FileId:    fileId,
 		Date:      date,
 		MimeType:  el.MimeType,
+		Size:      el.Size,
+		Digest:    el.Digest,
 	}, nil
 }
 
@@ -447,6 +461,8 @@ func (i *Index) ListFolder(path string) ([]FolderList, error) {
 				FileId:    fileId.String(),
 				Date:      date,
 				MimeType:  el.File.MimeType,
+				Size:      el.File.Size,
+				Digest:    el.File.Digest,
 			}
 			y++
 		} else {

index/index.go

@@ -16,7 +16,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
 
 /*
-//Build with:
+Build with:
 
 ```sh
 protoc \
@@ -40,10 +40,18 @@ option go_package = "github.com/ItalyPaleAle/index/proto";
 
 // IndexElement message
 message IndexElement {
+    // File's full path (directory + name)
     string path = 1 [json_name="p"];
+    // File ID (UUID), encoded as raw bytes
     bytes file_id = 2;
+    // Date when the file was added to the repository
     google.protobuf.Timestamp date = 3 [json_name="d"];
+    // File's content type
     string mime_type = 4 [json_name="m"];
+    // File size
+    int64 size = 5;
+    // File digest as SHA-256
+    bytes digest = 10;
 
     // This is added for compatibility with the version 1 JSON format
     string file_id_string = 102 [json_name="n",deprecated=true];

index/proto/index.pb.go

@@ -19,6 +19,7 @@ package repository
 
 import (
 	"context"
+	"crypto/sha256"
 	"errors"
 	"io"
 	"os"
@@ -31,6 +32,9 @@ import (
 	"github.com/gofrs/uuid"
 )
 
+// Flag that controls whether SHA-256 digests are calculated for files
+var CalculateDigest = true
+
 // AddStream adds a document to the repository by reading it from a stream
 func (repo *Repository) AddStream(ctx context.Context, in io.ReadCloser, filename, destinationFolder, mimeType string, size int64) (fileIdStr string, status int, err error) {
 	// Generate a file id
@@ -56,19 +60,28 @@ func (repo *Repository) AddStream(ctx context.Context, in io.ReadCloser, filenam
 		return "", RepositoryStatusExisting, nil
 	}
 
+	// Create the hash
+	hash := sha256.New()
+
+	// Tee the in stream into the hash
+	tee := io.TeeReader(in, hash)
+
 	// Write the data to an encrypted file
 	metadata := &crypto.Metadata{
 		Name:        sanitizedFilename,
 		ContentType: mimeType,
 		Size:        size,
 	}
-	_, err = repo.Store.Set(ctx, fileId.String(), in, nil, metadata)
+	_, err = repo.Store.Set(ctx, fileId.String(), tee, nil, metadata)
 	if err != nil {
 		return "", RepositoryStatusInternalError, err
 	}
 
+	// Complete the file's digest
+	digest := hash.Sum(nil)
+
 	// Add to the index
-	err = repo.Index.AddFile(sanitizedPath, fileId.Bytes(), mimeType)
+	err = repo.Index.AddFile(sanitizedPath, fileId.Bytes(), mimeType, size, digest)
 	if err != nil {
 		return "", RepositoryStatusInternalError, err
 	}