Commit c9ff6f5d authored by Niels Möller's avatar Niels Möller
Browse files

*** empty log message ***

Rev: src/rsync/.c-style:1.1
Rev: src/rsync/Makefile.am:1.1
Rev: src/rsync/README:1.1
Rev: src/rsync/generate.c:1.1
Rev: src/rsync/protocol.txt:1.1
Rev: src/rsync/receive.c:1.1
Rev: src/rsync/rsync.h:1.1
Rev: src/rsync/send.c:1.1
parent 47ecadbf
# Process this file with automake to produce Makefile.in
noinst_LIBRARIES = librsync.a
noinst_HEADERS = rsync.h
librsync_a_SOURCES = generate.c receive.c
This is some experimental stuff for using the rsync algorithm to
transfer files.
For protocol description, see protocol.txt.
generate.c id for generating and sending a list of checksums.
receive.c is for reconstructing a file at the receiver.
/* generate.c
*
* Generate the list of checksums.
*
* $Id$
*/
#include "rsync.h"
#include <assert.h>
#include <string.h>
/* block count, blocksize, tail */
#define HEADER_SIZE 12
/* weak sum, md5 sum */
#define ENTRY_SIZE 20
static void
rsync_init_block(struct rsync_generate_state *s)
{
s->a_sum = 0;
s->c_sum = 0;
md5_init(&s->block_sum);
}
static void
rsync_end_block(struct rsync_generate_state *s,
UINT8 *dst)
{
WRITE_UINT16(dst, s->a_sum);
WRITE_UINT16(dst + 2, s->c_sum);
md5_final(&s->block_sum);
md5_digest(&s->block_sum, dst + 4);
}
static void
rsync_output_block(struct rsync_generate_state *s)
{
assert(!s->buf_length);
if (s->avail_out < ENTRY_SIZE)
{
rsync_end_block(s, s->buf);
s->buf_length = ENTRY_SIZE;
s->buf_pos = 0;
}
else
{
rsync_end_block(s, s->next_out);
s->avail_out -= ENTRY_SIZE;
s->next_out += ENTRY_SIZE;
}
rsync_init_block(s);
}
/* Update checksums.
*
* For input x_i, i = 0, ..., l-1, we calculate (modulo 2^16)
*
* a_k = \sum_0^k x_i,
* b_k = \sum_0^k (l-i) x_i
*
* But in fact, we don't calculate all b_k, only the final
* value b_{l-1}, and we have the identity (by changing order of summation(
*
* b_{l-1} = \sum_0^{l-1} (l-i) x_i = \sum_0^{l-1} a_i
*
* So we keep track of the numbers c_k = \sum_0^k a_k rather than b_k.
*/
static void
rsync_update(struct rsync_generate_state *s,
UINT32 length)
{
unsigned i;
assert(length <= s->avail_in);
md5_update(&s->block_sum, s->next_in, length);
for (i = 0; i<length; i++)
{
s->a_sum += s->next_in[i] + RSYNC_CHAR_OFFSET;
s->c_sum += s->a_sum;
}
s->a_sum &= 0xffff;
s->c_sum &= 0xffff;
s->offset += length;
s->next_in += length;
s->avail_in -= length;
}
#define OUT(c) (*s->next_out++ = (c), --s->avail_out)
#define DONE (s->offset == s->total_length)
int rsync_generate(struct rsync_generate_state *s)
{
/* Have we maid any progress? */
int progress = 0;
for (;;)
{
if (s->avail_out && s->buf_length)
{
/* Output previously prepared data */
unsigned left = s->buf_length - s->buf_pos;
if (s->avail_out >= left)
{
memcpy(s->next_out, s->buf + s->buf_pos, left);
s->next_out += left;
s->avail_out -= left;
s->buf_length = 0;
progress = 1;
} else {
memcpy(s->next_out, s->buf + s->buf_pos, s->avail_out);
s->next_out += s->avail_out;
s->buf_pos += s->avail_out;
s->avail_out = 0;
return DONE ? RSYNC_DONE : RSYNC_PROGRESS;
}
}
if (!s->avail_out && s->buf_length)
/* We have buffered date, but no output space */
return progress ? RSYNC_PROGRESS : RSYNC_BUF_ERROR;
/* Here, the internal buffer should be flushed. */
assert(!s->buf_length);
if (DONE)
return s->avail_in ? RSYNC_INPUT_ERROR : RSYNC_DONE;
/* Now we have some output space. */
assert(s->left);
if (!s->avail_in)
return progress ? RSYNC_PROGRESS : RSYNC_BUF_ERROR;
/* And also some input to process. */
if (s->avail_in < s->left)
{
s->left -= s->avail_in;
rsync_update(s, s->avail_in);
return RSYNC_PROGRESS;
}
else
{
rsync_update(s, s->left);
rsync_output_block(s);
if ( (s->offset + s->block_size) > s->total_length)
{
/* Next block is the final one, and shorter than one block */
s->left = s->total_length - s->offset;
}
else
s->left = s->block_size;
progress = 1;
}
}
}
int
rsync_generate_init(struct rsync_generate_state *s,
UINT32 block_size,
UINT32 size)
{
/* Number of blocks */
unsigned count = (size + block_size - 1) / block_size;
unsigned tail = size % block_size;
s->block_size = block_size;
s->total_length = size;
s->offset = 0;
/* md5_init(&s->full_sum); */
WRITE_UINT32(s->buf, count);
WRITE_UINT32(s->buf + 4, block_size);
WRITE_UINT32(s->buf + 8, tail);
s->buf_length = HEADER_SIZE;
s->buf_pos = 0;
s->left = block_size;
return 1;
}
I think it is a good idea to chop the rsync protocol into orthogonal
parts. One part takes care of transport issues; i.e. version
negotiaon, authentication, compression. The other does the actual
data transfer insode a transport channel, and it doesn't have to care
about transport issues.
Through out this document, "receiver" is the party that wants to
reconstruct a copy of a file at the other pary, the "sender".
The data in the several phases is represented as follows:
Checksums sent from receiver and sender are represented as follows:
First
4 octets, number of blocks
4 octets, block size
4 octets, total size % blocksize, i.e. 0 or size of the final block
Next, a sequence of
4 octets of rolling checksum
16 octets md5 checksum
In rsync-2.4.1 md4 is used rather than md5. As the length header is
sent first, the file length must be known in advance.
Next, the sender transmits the information needed to reconstruct the
file. The stream consists of a series of "tokens". Either
4 octets, value 0 indicates end of file
or
4 octets, value n, MSB clear
n octets, literal data
or
4 octets, value n, MSB set. Let i = -(n + 1) == ~n (one's complent).
Then insert block i into the file.
(in rsync-2.4.1, the last form is more complicated; it refers to the
i:th previous token. I guess that the "previous token"-list is
initialized with the blocks of the original local file, but I'm not
quite sure. To me, it seems better to let the outer compressor do its
work).
Note that this token scheme seems to limit the number of blocks to
2^31-1, while the other phases of the protocol can handle up to 2^32-1
blocks.
At last, a checksum, to catch errors that might be caused, for
instance, if the files are modified under our feet.
16 octets, md5 checksum
Old versions of rsync doesn't send any checksum. More recent versions
do, but use md4 rather than md5.
First, an md5 checksum of the entire file (to catch
errors, for instance if the files are being updated under our feet),
16 octets md5 checksum of file
Next,
/* receive.c
*
* The receive end of the rsync algorithm.
*
* $Id$ */
#include "rsync.h"
#include <assert.h>
/* Reading a partial token */
#define STATE_TOKEN 0
/* Reading a literal */
#define STATE_LITERAL 1
/* Copying a local block */
#define STATE_LOOKUP 2
/* Reading final md5 sum */
#define STATE_CHECKSUM 3
/* Results in error */
#define STATE_INVALID 4
static void
rsync_update(struct rsync_receive_state *s,
UINT32 length)
{
md5_update(&s->full_sum, s->next_in, length);
s->next_in += length;
s->avail_in -= length;
}
#define GET() (assert(s->avail_in), s->avail_in--, *s->next_in++)
int
rsync_receive(struct rsync_receive_state *s)
{
int progress = 0;
for (;;)
switch (s->state)
{
do_token:
/* Here, i is octets read */
s->token = 0;
s->i = 0;
s->state = STATE_TOKEN;
case STATE_TOKEN:
if (!s->avail_in)
return progress ? RSYNC_PROGRESS : RSYNC_BUF_ERROR;
s->token = (s->token << 8) | GET();
progress = 1;
if (++s->i == 4)
{
if (!s->token)
goto do_checksum;
else if (! (s->token & 0x80000000))
{
s->i = s->token;
goto do_literal;
}
else
{
/* Index is one's complement */
s->token = -(s->token + 1);
goto do_lookup;
}
}
break;
do_literal:
/* Here, i is the number of octets to read. */
s->state = STATE_LITERAL;
case STATE_LITERAL:
{
UINT32 avail = MIN(s->avail_in, s->avail_out);
if (!avail)
return progress ? RSYNC_PROGRESS : RSYNC_BUF_ERROR;
if (avail < s->i)
{
memcpy(s->next_out, s->next_in, avail);
rsync_update(s, avail);
s->i -= avail;
}
else
{
memcpy(s->next_out, s->next_in, s->i);
rsync_update(s, s->i);
goto do_token;
}
}
break;
do_lookup:
s->state = STATE_LOOKUP;
s->i = 0;
case STATE_LOOKUP:
{
UINT32 done;
if (!s->avail_out)
return progress ? RSYNC_PROGRESS : RSYNC_BUF_ERROR;
switch (s->lookup(s->opaque, s->next_out, s->avail_out,
s->token, s->i, &done))
{
case 1:
rsync_update(s, done);
goto do_token;
case 0:
rsync_update(s, done);
s->i += done;
break;
case -1:
return RSYNC_INPUT_ERROR;
default:
assert(0);
}
}
break;
do_checksum:
/* i is number of octets read */
s->i = 0;
md5_final(&s->full_sum);
md5_digest(&s->full_sum, s->buf);
s->state = STATE_CHECKSUM;
case STATE_CHECKSUM:
if (!s->avail_in)
return progress ? RSYNC_PROGRESS : RSYNC_BUF_ERROR;
if (GET() != s->buf[s->i++])
return RSYNC_INPUT_ERROR;
if (s->i == MD5_DIGESTSIZE)
{
s->state = STATE_INVALID;
return RSYNC_DONE;
}
break;
default:
assert(0);
}
}
void
rsync_receive_init(struct rsync_receive_state *s)
{
s->state = STATE_TOKEN;
s->i = 0;
}
/* rsync.h
*
* $Id$
*/
/*
Copyright (C) Andrew Tridgell 1996
Copyright (C) Paul Mackerras 1996
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
/* Hacked by Niels Möller */
#ifndef RSYNC_H_INCLUDED
#define RSYNC_H_INCLUDED
#if LSH
# include "lsh_types.h"
#else
# if HAVE_CONFIG_H
# include "config.h"
# endif
#endif
#include "md5.h"
#include <stdlib.h>
/* FIXME: replace with proper autoconf check */
#define OFF_T size_t
#define RSYNC_SUM_LENGTH MD5_DIGESTSIZE
/* Constant used in checksum calculation */
#define RSYNC_CHAR_OFFSET 0
struct rsync_sum_buf
{
OFF_T offset; /* offset in file of this chunk */
unsigned len; /* length of chunk of file */
unsigned i; /* index of this chunk */
UINT32 sum1; /* simple checksum */
char sum2[RSYNC_SUM_LENGTH]; /* checksum */
};
/* Initial checksum calculations (by the receiver) */
#define RSYNC_INTERNAL_BUF_SIZE 20
/* NOTE: Unlike zlib, we want to know the file size before we start.
* This could be relxed, but requires some modifications to the
* protocol. */
struct rsync_generate_state
{
/* Public fields */
UINT8 *next_in;
UINT32 avail_in;
UINT8 *next_out;
UINT32 avail_out;
UINT32 block_size;
UINT32 total_length;
UINT32 offset; /* Current offset in input file. */
/* Weak check sum */
unsigned a_sum;
unsigned c_sum;
struct md5_ctx block_sum;
/* Internal state */
UINT8 buf[RSYNC_INTERNAL_BUF_SIZE];
UINT8 buf_length; /* Zero means no buffered data. */
UINT8 buf_pos;
UINT32 left; /* Amount left of current block */
};
/* Return values */
/* Things are working fine */
#define RSYNC_PROGRESS 0
/* All data is flushed to the output */
#define RSYNC_DONE 1
/* No progress possible */
#define RSYNC_BUF_ERROR 2
/* Invalid input */
#define RSYNC_INPUT_ERROR 3
int rsync_generate(struct rsync_generate_state *state);
int rsync_generate_init(struct rsync_generate_state *state,
UINT32 block_size,
UINT32 size);
/* Receiving a file. */
/* The receiver calls this function to copy at most LENGTH octets of
* local data to the output buffer.
*
* OPAQUE is state private to the lookup function. DST and LENGTH give
* the location of the destination buffer. INDEX is the block to read,
* and OFFSET is a location within that block.
*
* The function should return
*
* -1 on failure (and it has to check INDEX and OFFSET for validity).
* 0 if copying succeeds, but not all of the block was copied.
* 1 if copying succeeds, and the final octet of the data swas copied.
*
* On success, the function should set *DONE to the amount of data copied.
*/
typedef int (*rsync_lookup_read_t)(void *opaque,
UINT8 *dst, UINT32 length,
UINT32 index, UINT32 offset, UINT32 *done);
enum rsync_receive_mode;
struct rsync_receive_state
{
/* Public fields */
UINT8 *next_in;
UINT32 avail_in;
UINT8 *next_out;
UINT32 avail_out;
UINT32 block_size;
/* UINT32 offset; */ /* Current offset in output file. */
rsync_lookup_read_t lookup;
void *opaque;
struct md5_ctx full_sum; /* Sum of all input data */
/* Private state */
int state;
UINT32 token;
UINT32 i;
UINT8 buf[MD5_DIGESTSIZE];
};
int rsync_receive(struct rsync_receive_state *state);
void rsync_receive_init(struct rsync_receive_state *state);
#endif /* RSYNC_H_INCLUDED */
/* send.c
*
* The sending end of the rsync algorithm.
*
* $Id$ */
#include "rsync.h"
#include <assert.h>
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment