[Unison-hackers] [patch] high system time with BTRFS and compression...

[Benjamin Pierce]

Hi Daniel,

All this code has been substantially rewritten since 2.27 (so there's nowhere to apply your patch any more) -- should perform even better now!

   - Benjamin



On May 28, 2010, at 10:13 AM, Daniel J Blueman wrote:

> With unison 2.27.57 on Linux (Ubuntu 10.04 x86-64, client and server),
> performing synchronisation with a BTRFS filesystem with compression
> enabled takes higher system time, due to many threads performing
> smaller (~4K) synchronous reads.
> 
> Increasing the buffer size to 64K helps alleviate this, and will
> provide a smaller win on other filesystems. Also, on faster networks
> with larger MTU, 64K read/write to the socket provides more
> throughput.
> 
> Please consider and thanks,
>  Daniel
> 
> diff -ur unison-2.27.57/remote.ml unison-2.27.57-new/remote.ml
> --- unison-2.27.57/remote.ml    2008-01-18 13:53:35.000000000 +0000
> +++ unison-2.27.57-new/remote.ml        2010-05-26 12:49:00.146269467 +0100
> @@ -79,7 +79,7 @@
>  let receivedBytes = ref 0.
>  let emittedBytes = ref 0.
> 
> -let inputBuffer_size = 8192
> +let inputBuffer_size = 65536
> 
>  let fill_inputBuffer conn =
>   assert (conn.inputLength = 0);
> @@ -127,7 +127,7 @@
> 
>  (****)
> 
> -let outputBuffer_size = 8192
> +let outputBuffer_size = 65536
> 
>  let rec send_output conn =
>   catch_io_errors
> diff -ur unison-2.27.57/transfer.ml unison-2.27.57-new/transfer.ml
> --- unison-2.27.57/transfer.ml  2007-04-02 04:03:20.000000000 +0100
> +++ unison-2.27.57-new/transfer.ml      2010-05-26 12:51:12.676894227 +0100
> @@ -234,8 +234,8 @@
>  let send infd length showProgress transmit =
>   debug (fun() -> Util.msg "sending file\n");
>   let timer = Trace.startTimer "Sending file using generic transmission" in
> -  let bufSz = 8192 in
> -  let bufSzFS = Uutil.Filesize.ofInt 8192 in
> +  let bufSz = 65536 in
> +  let bufSzFS = Uutil.Filesize.ofInt 65536 in
>   let buf = String.create bufSz in
>   let q = makeQueue length in
>   let rec sendSlice length =
> @@ -298,14 +298,14 @@
>   (*** PREPROCESS ***)
> 
>   (* Preprocess buffer size *)
> -  let preproBufSize = 8192
> +  let preproBufSize = 65536
> 
>   (* Incrementally build arg by executing f on successive blocks (of size
>      'blockSize') of the input stream (pointed by 'infd').
>      The procedure uses a buffer of size 'bufferSize' to load the input,
>      and eventually handles the buffer update. *)
>   let blockIter infd f arg maxCount =
> -    let bufferSize = 8192 + blockSize in
> +    let bufferSize = 65536 + blockSize in
>     let buffer = String.create bufferSize in
>     let rec iter count arg offset length =
>       if count = maxCount then arg else begin
> @@ -363,7 +363,7 @@
>   (*** DECOMPRESSION ***)
> 
>   (* Decompression buffer size *)
> -  let decomprBufSize = 8192
> +  let decomprBufSize = 65536
> 
>   (* For each transfer instruction, either output a string or copy one or
>      several blocks from the old file. *)
> @@ -517,8 +517,8 @@
> 
>   (* Compression buffer size *)
>   (* MUST be >= 2 * blockSize *)
> -  let comprBufSize = 8192
> -  let comprBufSizeFS = Uutil.Filesize.ofInt 8192
> +  let comprBufSize = 65536
> +  let comprBufSizeFS = Uutil.Filesize.ofInt 65536
> 
>   (* Compress the file using the algorithm described in the header *)
>   let rsyncCompress sigs infd srcLength showProgress transmit =
> -- 
> Daniel J Blueman