Skip to content

Commit 640fa9f

Browse files
ItalyPaleAleItalyPaleAle
authored
Updated app and APIs to use index transactions
1 parent 05d330c commit 640fa9f

15 files changed

+143
-52
lines changed

‎cmd/add.go

+12
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,12 @@ You must specify a destination, which is a folder inside the repository where yo
114114
}
115115
repo.Index.SetProvider(indexProvider)
116116

117+
// Start a transaction with the index to add all files
118+
err = repo.BeginTransaction()
119+
if err != nil {
120+
return NewExecError(ErrorApp, "Error starting a transaction", err)
121+
}
122+
117123
// Iterate through the args and add them all
118124
ctx := context.Background()
119125
res := make(chan repository.PathResultMessage)
@@ -156,6 +162,12 @@ You must specify a destination, which is a folder inside the repository where yo
156162
}
157163
}
158164

165+
// End the transaction
166+
err = repo.CommitTransaction()
167+
if err != nil {
168+
return NewExecError(ErrorApp, "Error committing a transaction", err)
169+
}
170+
159171
return nil
160172
},
161173
}

‎cmd/ls.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ Shows the list of all files and folders contained in the repository at a given p
9191
}
9292

9393
// Get the list of files in the folder
94-
list, err := idx.ListFolder(path)
94+
list, err := idx.ListFolder(0, path)
9595
if err != nil {
9696
return NewExecError(ErrorApp, "Error listing the contents of the folder", err)
9797
}

‎cmd/repo-info.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ func NewRepoInfoCmd() *cobra.Command {
8080
idx.SetProvider(indexProvider)
8181

8282
// Get stats
83-
stat, err = idx.Stat()
83+
stat, err = idx.Stat(0)
8484
if err != nil {
8585
return NewExecError(ErrorApp, "Could not get the stats from the repository", err)
8686
}

‎cmd/rm.go

+12
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,12 @@ To remove a file, specify its exact path. To remove a folder recursively, specif
9292
}
9393
repo.Index.SetProvider(indexProvider)
9494

95+
// Start a transaction with the index to remove all files
96+
err = repo.BeginTransaction()
97+
if err != nil {
98+
return NewExecError(ErrorApp, "Error starting a transaction", err)
99+
}
100+
95101
// Iterate through the args and remove all files
96102
res := make(chan repository.PathResultMessage)
97103
go func() {
@@ -116,6 +122,12 @@ To remove a file, specify its exact path. To remove a folder recursively, specif
116122
}
117123
}
118124

125+
// End the transaction
126+
err = repo.CommitTransaction()
127+
if err != nil {
128+
return NewExecError(ErrorApp, "Error committing a transaction", err)
129+
}
130+
119131
return nil
120132
},
121133
}

‎index/index.go

+8-3
Original file line numberDiff line numberDiff line change
@@ -130,15 +130,20 @@ func (i *Index) CommitTransaction(tx IndexTxId) error {
130130
}
131131

132132
// Refresh an index if necessary
133-
func (i *Index) Refresh(tx IndexTxId) error {
133+
func (i *Index) Refresh(tx IndexTxId, force bool) error {
134134
// Semaphore - can be skipped if we're in the transaction
135135
if i.tx == 0 || i.tx != tx {
136136
i.semaphore.Lock()
137137
defer i.semaphore.Unlock()
138138
}
139139

140-
err := i.refresh()
141-
return err
140+
// We can force a refresh by deleting the cache first
141+
if force {
142+
i.elements = make([]*pb.IndexElement, 0)
143+
}
144+
145+
// Do the refresh
146+
return i.refresh()
142147
}
143148

144149
// Internal function that performs the refresh

‎index/index_test.go

+2-3
Original file line numberDiff line numberDiff line change
@@ -188,9 +188,8 @@ func TestIndex(t *testing.T) {
188188
checkSaved(t, []int{0}, 1)
189189
tx = 0
190190

191-
// Re-set the provider and trigger a refresh to have the index re-built
192-
i.SetProvider(provider)
193-
err = i.Refresh(0)
191+
// Trigger a forced refresh to have the index re-built
192+
err = i.Refresh(0, true)
194193
assert.NoError(t, err)
195194
checkIndex(t)
196195

‎repository/add.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ func (repo *Repository) AddStream(ctx context.Context, in io.ReadCloser, filenam
5151
mimeType = utils.SanitizeMimeType(mimeType)
5252

5353
// Check if the file exists in the index already
54-
existing, err := repo.Index.GetFileByPath(sanitizedPath)
54+
existing, err := repo.Index.GetFileByPath(repo.tx, sanitizedPath)
5555
if err != nil {
5656
return "", RepositoryStatusInternalError, err
5757
}
@@ -85,7 +85,7 @@ func (repo *Repository) AddStream(ctx context.Context, in io.ReadCloser, filenam
8585
if existing != nil {
8686
// Remove it from the index
8787
var objs []string
88-
objs, _, err = repo.Index.DeleteFile(sanitizedPath)
88+
objs, _, err = repo.Index.DeleteFile(repo.tx, sanitizedPath)
8989
if err != nil {
9090
return "", RepositoryStatusInternalError, err
9191
}
@@ -100,7 +100,7 @@ func (repo *Repository) AddStream(ctx context.Context, in io.ReadCloser, filenam
100100
}
101101

102102
// Add to the index
103-
err = repo.Index.AddFile(sanitizedPath, fileId.Bytes(), mimeType, size, digest, force)
103+
err = repo.Index.AddFile(repo.tx, sanitizedPath, fileId.Bytes(), mimeType, size, digest, force)
104104
if err != nil {
105105
return "", RepositoryStatusInternalError, err
106106
}

‎repository/remove.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import (
2626
// RemovePath removes a path by its prefix, and reports each element removed in the res channel
2727
func (repo *Repository) RemovePath(ctx context.Context, path string, res chan<- PathResultMessage) {
2828
// Remove from the index and get the list of objects to delete
29-
objects, paths, err := repo.Index.DeleteFile(path)
29+
objects, paths, err := repo.Index.DeleteFile(repo.tx, path)
3030
if err != nil {
3131
status := RepositoryStatusInternalError
3232
errStr := err.Error()

‎repository/repository.go

+33-6
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
1818
package repository
1919

2020
import (
21+
"errors"
22+
2123
"github.com/ItalyPaleAle/prvt/fs"
2224
"github.com/ItalyPaleAle/prvt/index"
2325
)
@@ -38,16 +40,41 @@ const (
3840
RepositoryStatusUserError
3941
)
4042

41-
// Repository is the object that manages the repository
42-
type Repository struct {
43-
Store fs.Fs
44-
Index *index.Index
45-
}
46-
4743
// PathResultMessage is the message passed to the res channel in AddPath/RemovePath
4844
type PathResultMessage struct {
4945
Path string
5046
Status int
5147
Err error
5248
FileId string
5349
}
50+
51+
// Repository is the object that manages the repository
52+
type Repository struct {
53+
Store fs.Fs
54+
Index *index.Index
55+
56+
tx index.IndexTxId
57+
}
58+
59+
// BeginTransaction starts a transaction to add or remove multiple files at once
60+
func (repo *Repository) BeginTransaction() error {
61+
if repo.Index == nil {
62+
return errors.New("index is not defined")
63+
}
64+
65+
// Begin a transaction with the index
66+
repo.tx = repo.Index.BeginTransaction()
67+
return nil
68+
}
69+
70+
// CommitTransaction commits a transaction and saves pending changes
71+
func (repo *Repository) CommitTransaction() error {
72+
if repo.tx == 0 {
73+
return errors.New("no active transaction")
74+
}
75+
76+
// Begin a transaction with the index
77+
err := repo.Index.CommitTransaction(repo.tx)
78+
repo.tx = 0
79+
return err
80+
}

‎server/api-delete-tree.go

+14
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,13 @@ func (s *Server) DeleteTreeHandler(c *gin.Context) {
3737
path = "/" + path
3838
}
3939

40+
// Start a transaction
41+
err := s.Repo.BeginTransaction()
42+
if err != nil {
43+
c.AbortWithError(http.StatusInternalServerError, err)
44+
return
45+
}
46+
4047
// Remove the path
4148
res := make(chan repository.PathResultMessage)
4249
go func() {
@@ -66,5 +73,12 @@ func (s *Server) DeleteTreeHandler(c *gin.Context) {
6673
response = append(response, r)
6774
}
6875

76+
// Commit the transaction
77+
err = s.Repo.CommitTransaction()
78+
if err != nil {
79+
c.AbortWithError(http.StatusInternalServerError, err)
80+
return
81+
}
82+
6983
c.JSON(http.StatusOK, response)
7084
}

‎server/api-get-info.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func (s *Server) GetInfoHandler(c *gin.Context) {
4949

5050
// If the repo is unlocked, add stats too
5151
if res.RepoUnlocked {
52-
stat, err := s.Repo.Index.Stat()
52+
stat, err := s.Repo.Index.Stat(0)
5353
if err != nil {
5454
c.AbortWithError(http.StatusInternalServerError, err)
5555
return

‎server/api-get-metadata.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,10 @@ func (s *Server) GetMetadataHandler(c *gin.Context) {
4747
fileIdUUID, err := uuid.FromString(file)
4848
// Check if we have a file ID
4949
if err == nil && fileIdUUID.Version() == 4 {
50-
el, err = s.Repo.Index.GetFileById(file)
50+
el, err = s.Repo.Index.GetFileById(0, file)
5151
} else {
5252
// Re-add the leading /
53-
el, err = s.Repo.Index.GetFileByPath("/" + file)
53+
el, err = s.Repo.Index.GetFileByPath(0, "/"+file)
5454
}
5555
if err != nil {
5656
c.AbortWithError(http.StatusInternalServerError, err)

‎server/api-get-tree.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ func (s *Server) GetTreeHandler(c *gin.Context) {
3636
}
3737

3838
// Get the list of files in the folder
39-
list, err := s.Repo.Index.ListFolder(path)
39+
list, err := s.Repo.Index.ListFolder(0, path)
4040
if err != nil {
4141
c.AbortWithError(http.StatusInternalServerError, err)
4242
return

‎server/api-post-tree.go

+47-25
Original file line numberDiff line numberDiff line change
@@ -50,39 +50,54 @@ func (s *Server) PostTreeHandler(c *gin.Context) {
5050
path += "/"
5151
}
5252

53-
// Create a channel to listen to the responses
54-
res := make(chan repository.PathResultMessage)
55-
go func() {
56-
defer close(res)
53+
// Get the data from the request body, which must be a multipart/form-data
54+
mpf, err := c.MultipartForm()
55+
if err != nil {
56+
c.AbortWithError(http.StatusBadRequest, err)
57+
return
58+
}
5759

58-
// Get the data from the request body, which must be a multipart/form-data
59-
mpf, err := c.MultipartForm()
60-
if err != nil {
61-
c.AbortWithError(http.StatusBadRequest, err)
62-
return
63-
}
60+
// Check if we're forcing the request
61+
force := false
62+
forceVal, ok := mpf.Value["force"]
63+
if ok && len(forceVal) > 0 {
64+
force = utils.IsTruthy(forceVal[0])
65+
}
6466

65-
// Check if we're forcing the request
66-
force := false
67-
forceVal, ok := mpf.Value["force"]
68-
if ok && len(forceVal) > 0 {
69-
force = utils.IsTruthy(forceVal[0])
70-
}
67+
// Create a channel to listen to the responses
68+
res := make(chan repository.PathResultMessage)
7169

72-
// Check if we have a path from the local filesystem or a file uploaded
73-
uploadFiles := mpf.File["file"]
74-
localPaths := mpf.Value["localpath"]
75-
if localPaths != nil && len(localPaths) > 0 && (uploadFiles == nil || len(uploadFiles) == 0) {
70+
// Check if we have a path from the local filesystem or a file uploaded
71+
uploadFiles := mpf.File["file"]
72+
localPaths := mpf.Value["localpath"]
73+
var method func()
74+
if localPaths != nil && len(localPaths) > 0 && (uploadFiles == nil || len(uploadFiles) == 0) {
75+
method = func() {
7676
s.addLocalPath(ctx, localPaths, path, force, res)
77-
} else if uploadFiles != nil && len(uploadFiles) > 0 && (localPaths == nil || len(localPaths) == 0) {
77+
}
78+
} else if uploadFiles != nil && len(uploadFiles) > 0 && (localPaths == nil || len(localPaths) == 0) {
79+
method = func() {
7880
s.addUploadedFiles(ctx, uploadFiles, path, force, res)
79-
} else {
80-
c.AbortWithError(http.StatusBadRequest, errors.New("need to specify one and only one of 'file' or 'localpath' form fields"))
81-
return
8281
}
82+
} else {
83+
c.AbortWithError(http.StatusBadRequest, errors.New("need to specify one and only one of 'file' or 'localpath' form fields"))
84+
return
85+
}
86+
87+
// Start a transaction
88+
err = s.Repo.BeginTransaction()
89+
if err != nil {
90+
c.AbortWithError(http.StatusInternalServerError, err)
91+
return
92+
}
93+
94+
// Add the file(s) in a goroutine
95+
go func() {
96+
method()
97+
close(res)
8398
}()
8499

85-
// Response
100+
// Read each response from the channel
86101
response := make([]TreeOperationResponse, 0)
87102
for el := range res {
88103
r := TreeOperationResponse{
@@ -106,6 +121,13 @@ func (s *Server) PostTreeHandler(c *gin.Context) {
106121
response = append(response, r)
107122
}
108123

124+
// Commit the transaction
125+
err = s.Repo.CommitTransaction()
126+
if err != nil {
127+
c.AbortWithError(http.StatusInternalServerError, err)
128+
return
129+
}
130+
109131
c.JSON(http.StatusOK, response)
110132
}
111133

‎wasm/index.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ func (i *RepoIndex) Refresh() js.Func {
104104

105105
// Run in a goroutine because this might make a blocking call
106106
go func() {
107-
err := i.index.Refresh(force)
107+
err := i.index.Refresh(0, force)
108108
if err != nil {
109109
reject.Invoke(jsError(err.Error()))
110110
return
@@ -150,7 +150,7 @@ func (i *RepoIndex) Stat() js.Func {
150150

151151
// Run in a goroutine because this might make a blocking call
152152
go func() {
153-
stats, err := i.index.Stat()
153+
stats, err := i.index.Stat(0)
154154
if err != nil {
155155
reject.Invoke(jsError(err.Error()))
156156
return
@@ -187,7 +187,7 @@ func (i *RepoIndex) GetFileByPath() js.Func {
187187

188188
// Run in a goroutine because this might make a blocking call
189189
go func() {
190-
file, err := i.index.GetFileByPath(path)
190+
file, err := i.index.GetFileByPath(0, path)
191191
if err != nil {
192192
reject.Invoke(jsError(err.Error()))
193193
return
@@ -225,7 +225,7 @@ func (i *RepoIndex) GetFileById() js.Func {
225225

226226
// Run in a goroutine because this might make a blocking call
227227
go func() {
228-
file, err := i.index.GetFileById(fileId)
228+
file, err := i.index.GetFileById(0, fileId)
229229
if err != nil {
230230
reject.Invoke(jsError(err.Error()))
231231
return
@@ -307,7 +307,7 @@ func (i *RepoIndex) ListFolder() js.Func {
307307

308308
// Run in a goroutine because this might make a blocking call
309309
go func() {
310-
list, err := i.index.ListFolder(path)
310+
list, err := i.index.ListFolder(0, path)
311311
if err != nil {
312312
reject.Invoke(jsError(err.Error()))
313313
return

0 commit comments

Comments
 (0)