Skip to content

Fix File#truncate and #lock for Win32 append-mode files #14706

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions spec/std/file_spec.cr
Original file line number Diff line number Diff line change
Expand Up @@ -1049,6 +1049,41 @@ describe "File" do
end
end

it "does not overwrite existing content in append mode" do
with_tempfile("append-override.txt") do |filename|
File.write(filename, "0123456789")

File.open(filename, "a") do |file|
file.seek(5)
file.write "abcd".to_slice
end

File.read(filename).should eq "0123456789abcd"
end
end

it "truncates file opened in append mode (#14702)" do
with_tempfile("truncate-append.txt") do |path|
File.write(path, "0123456789")

File.open(path, "a") do |file|
file.truncate(4)
end

File.read(path).should eq "0123"
end
end

it "locks file opened in append mode (#14702)" do
with_tempfile("truncate-append.txt") do |path|
File.write(path, "0123456789")

File.open(path, "a") do |file|
file.flock_exclusive { }
end
end
end

it "can navigate with pos" do
File.open(datapath("test_file.txt")) do |file|
file.pos = 3
Expand Down
3 changes: 3 additions & 0 deletions src/crystal/system/unix/file.cr
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ module Crystal::System::File
{fd, fd < 0 ? Errno.value : Errno::NONE}
end

protected def system_set_mode(mode : String)
end

def self.info?(path : String, follow_symlinks : Bool) : ::File::Info?
stat = uninitialized LibC::Stat
if follow_symlinks
Expand Down
3 changes: 3 additions & 0 deletions src/crystal/system/wasi/file.cr
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ require "../unix/file"

# :nodoc:
module Crystal::System::File
protected def system_set_mode(mode : String)
end

def self.chmod(path, mode)
raise NotImplementedError.new "Crystal::System::File.chmod"
end
Expand Down
27 changes: 23 additions & 4 deletions src/crystal/system/win32/file.cr
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ require "c/ntifs"
require "c/winioctl"

module Crystal::System::File
@system_append = false

def self.open(filename : String, mode : String, perm : Int32 | ::File::Permissions) : FileDescriptor::Handle
perm = ::File::Permissions.new(perm) if perm.is_a? Int32
# Only the owner writable bit is used, since windows only supports
Expand Down Expand Up @@ -52,10 +54,9 @@ module Crystal::System::File
LibC::FILE_GENERIC_READ
end

if flags.bits_set? LibC::O_APPEND
access |= LibC::FILE_APPEND_DATA
access &= ~LibC::FILE_WRITE_DATA
end
# do not handle `O_APPEND`, because Win32 append mode relies on removing
# `FILE_WRITE_DATA` which breaks file truncation and locking; instead,
# simply set the end of the file as the write offset in `#write_blocking`

if flags.bits_set? LibC::O_TRUNC
if flags.bits_set? LibC::O_CREAT
Expand Down Expand Up @@ -96,6 +97,24 @@ module Crystal::System::File
{access, disposition, attributes}
end

protected def system_set_mode(mode : String)
@system_append = true if mode.starts_with?('a')
end

private def write_blocking(handle, slice)
if @system_append
write_blocking(handle, slice) do
overlapped = LibC::OVERLAPPED.new
overlapped.union.offset.offset = 0xFFFFFFFF_u32
overlapped.union.offset.offsetHigh = 0xFFFFFFFF_u32
ret = LibC.WriteFile(handle, slice, slice.size, out bytes_written, pointerof(overlapped))
{ret, bytes_written}
end
else
super
end
end
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thought: Maybe we could reduce this duplication? The lib call is always the same, except for the overlapped struct.
I suppose it should be possible to have only a single implementation of write_blocking with an additional parameter for append mode (could be a boolean flag, or the offset, or the entire overlapped struct).

module Crystal::System::FileDescriptor
  private def write_blocking(handle, slice, append = false)
    if append
      overlapped = LibC::OVERLAPPED.new
      overlapped.union.offset.offset = 0xFFFFFFFF_u32
      overlapped.union.offset.offsetHigh = 0xFFFFFFFF_u32
    end
    write_blocking(handle, slice) do
      ret = LibC.WriteFile(handle, slice, slice.size, out bytes_written, pointerof(overlapped))
      {ret, bytes_written}
    end
  end
end

module Crystal::System::File
  private def write_blocking(handle, slice)
    write_blocking(handle, slice, @system_append)
  end
end

This should make the diff smaller, the code has less duplication and is easier to comprehend.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could also consider moving @system_append to Crystal::System::FileDescriptor because that's where it matters. Might also make sense to expose it in FileDescriptor.new to use this behaviour with an existing handle.

Copy link
Contributor Author

@HertzDevil HertzDevil Jun 13, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That snippet wouldn't work as pointerof(overlapped) should not be a Pointer(LibC::OVERLAPPED | Nil). Always passing an overlapped doesn't work either as that makes all non-append writes occur at offset 0

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're probably right that IO::FileDescriptor could open a regular file via its #fd directly and it would lose the append mode. I'm not sure if there is a way around that

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry, I scribbled the code down too quickly. pointerof(overlapped) should go in the if branch, of course.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're probably right that IO::FileDescriptor could open a regular file via its #fd directly and it would lose the append mode. I'm not sure if there is a way around that

There's probably no way around it. We could try to be smart and query the access mode of the file descriptor to figure out the intention. But we cannot change it anyway, so it's probably a mute point.
It's probably fine to accept that an external file descriptor without FILE_WRITE_DATA access mode won't be able to truncate or lock.
But we can offer an option for a file descriptor with FILE_WRITE_DATA to only append to the end of the file if we allow to configure @system_access in the constructor.


NOT_FOUND_ERRORS = {
WinError::ERROR_FILE_NOT_FOUND,
WinError::ERROR_PATH_NOT_FOUND,
Expand Down
11 changes: 9 additions & 2 deletions src/crystal/system/win32/file_descriptor.cr
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ module Crystal::System::FileDescriptor
end
end

private def write_blocking(handle, slice)
ret = LibC.WriteFile(handle, slice, slice.size, out bytes_written, nil)
private def write_blocking(handle, slice, &)
ret, bytes_written = yield
if ret.zero?
case error = WinError.value
when .error_access_denied?
Expand All @@ -69,6 +69,13 @@ module Crystal::System::FileDescriptor
bytes_written
end

private def write_blocking(handle, slice)
write_blocking(handle, slice) do
ret = LibC.WriteFile(handle, slice, slice.size, out bytes_written, nil)
{ret, bytes_written}
end
end

# :nodoc:
def system_blocking?
@system_blocking
Expand Down
2 changes: 1 addition & 1 deletion src/file.cr
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ class File < IO::FileDescriptor
def self.new(filename : Path | String, mode = "r", perm = DEFAULT_CREATE_PERMISSIONS, encoding = nil, invalid = nil, blocking = true)
filename = filename.to_s
fd = Crystal::System::File.open(filename, mode, perm: perm)
new(filename, fd, blocking: blocking, encoding: encoding, invalid: invalid)
new(filename, fd, blocking: blocking, encoding: encoding, invalid: invalid).tap { |f| f.system_set_mode(mode) }
end

getter path : String
Expand Down
Loading