buffer.cmod 76.4 KB
Newer Older
1 2 3 4 5 6
/* -*- c -*-
|| This file is part of Pike. For copyright information see COPYRIGHT.
|| Pike is distributed under GPL, LGPL and MPL. See the file COPYING
|| for more information.
*/

7
#include "global.h"
8 9
#include "fdlib.h"
#include "pike_netlib.h"
10 11 12 13 14 15
#include "object.h"
#include "interpret.h"
#include "operators.h"
#include "bignum.h"
#include "sscanf.h"
#include "builtin_functions.h"
16
#include "cyclic.h"
17 18 19 20
#include "backend.h"
#include "fd_control.h"
#include "file_machine.h"
#include "file.h"
21 22
#include "whitespace.h"
#include "pike_types.h"
23
#include "pike_threadlib.h"
24
#include "buffer.h"
Per Hedbor's avatar
Per Hedbor committed
25
#include "module_support.h"
26
#include "bitvector.h"
27
#include "pike_search.h"
28
#include "sprintf.h"
29

30
#ifdef HAVE_ARPA_INET_H
31
#include <arpa/inet.h>
32
#endif /* HAVE_ARPA_INET_H */
33

34 35
#define READ_CHUNKSIZE		32768
#define WRITE_CHUNKSIZE		32768
36

37 38 39
#define DEFAULT_CMOD_STORAGE static
DECLARATIONS

Per Hedbor's avatar
Per Hedbor committed
40 41 42 43
struct sysmem {
  unsigned char *p;
  size_t size;
};
44

Per Hedbor's avatar
Per Hedbor committed
45
static struct program *buffer_error_program;
46

47
/*! @module _Stdio
48 49
 */

50 51 52 53 54
/* Remap to not clash with String.Buffer. */
#define     PROG_BUFFER_ID     PROG_STDIO_BUFFER_ID
#define tObjImpl_BUFFER	   tObjImpl_STDIO_BUFFER
#define tObjIs_BUFFER	     tObjIs_STDIO_BUFFER

55
/*! @class Buffer
56 57 58 59 60 61 62
 *!
 *! A buffer to use as input or buffering when doing I/O. It is
 *! similar to @[String.Buffer], but can only contain 8bit data and is
 *! designed for protocol parsing. It is optimized for reading from
 *! the beginning and adding to the end, and will try to minimize the
 *! amount of data copying that is done.
 *!
63 64 65 66 67 68 69 70 71 72
 *! The class maintains two separate offsets, one for reading and one
 *! for writing. The functions that add data all do so at the write
 *! offset (the end of the buffer), and reading is done from the read
 *! offset (the start of the buffer).
 *!
 *! The class can also be used to directly read from and write to
 *! filedescriptors if so desired. This eliminates at least one memory
 *! copy.
 *!
 *! @note
73
 *!  The "avoid copy" part means that a Buffer will never shrink
74 75
 *!  unless you call the @[trim] function.
 *!
76
 */
77
PIKECLASS Buffer
78 79 80 81 82
  /* This is needed to get consisten behavior in case this
   * buffer is a sub-buffer, in which case the source
   * buffer needs to be unlocked as soon as possible.
   */
  program_flags PROGRAM_DESTRUCT_IMMEDIATE;
83
{
84 85 86 87 88
#if PRECOMPILE_API_VERSION > 5
  PIKEVAR int b.num_malloc;
  PIKEVAR int b.num_move;
#endif

89
  CVAR Buffer b;
90

91 92 93 94 95 96 97
  EXTRA
  {
    PIKE_MAP_VARIABLE("__output", OFFSETOF(Buffer_struct, b.output),
		      tMix, PIKE_T_MIXED, ID_PRIVATE|ID_HIDDEN|ID_PROTECTED);
  }


98
  static void io_set_error_mode( Buffer *io, struct program *m )
99
  {
100
    if( m ) add_ref(m);
Per Hedbor's avatar
Per Hedbor committed
101 102
    if( io->error_mode ) free_program( io->error_mode );
    io->error_mode = m;
103 104
  }

105 106
  PMOD_EXPORT Buffer *io_buffer_from_object(struct object *o) {
    return get_storage(o, Buffer_program);
107 108
  }

109

110
  static void io_unlock( Buffer *io )
111 112 113 114
  {
    io->locked--;
  }

115
  static void io_lock( Buffer *io )
116 117 118 119
  {
    io->locked++;
  }

120 121 122 123
  static void io_was_locked( )
    ATTRIBUTE((noclone,noinline));

  static void io_was_locked( )
124 125 126 127 128
  {
      Pike_error("Can not modify the buffer right now, "
                 " there are active subbuffers.\n");
  }

129
  static void io_ensure_unlocked(Buffer *io)
130 131 132 133 134
  {
    if( io->locked )
      io_was_locked( );
  }

135
  PMOD_EXPORT void io_trim( Buffer *io )
Per Hedbor's avatar
Per Hedbor committed
136 137
    ATTRIBUTE((noinline));

Arne Goedeke's avatar
Arne Goedeke committed
138 139 140 141 142 143
  static void io_trim_waste( Buffer *io )
  {
    if( UNLIKELY(io->allocated > (io_len(io) * (1.0+io->max_waste))) )
        io_trim(io);
  }

144
  static int io_is_whitespace( Buffer *io, size_t pos )
Per Hedbor's avatar
Per Hedbor committed
145
  {
Per Hedbor's avatar
Per Hedbor committed
146 147 148 149 150 151 152 153
    if( pos > io_len( io ) )
      return -1;
    switch( io->buffer[io->offset+pos] )
    {
      SPACECASE8
        return 1;
    }
    return 0;
Per Hedbor's avatar
Per Hedbor committed
154 155
  }

156 157 158 159 160 161 162 163 164 165 166
  static void io_discard_bufstart( Buffer *io )
  {
      if ( LIKELY(!io->locked_move) )
      {
          memmove( io->buffer, io_read_pointer(io), io_len(io) );
          io->len -= io->offset;
          io->num_move++;
          io->offset = 0;
      }
  }

167
  PMOD_EXPORT void io_trim( Buffer *io )
Per Hedbor's avatar
Per Hedbor committed
168
  {
169
      if( io->malloced && (io->offset > 64 || io->len > 64) && !io->locked)
Per Hedbor's avatar
Per Hedbor committed
170
      {
171 172 173
          if( io->offset > 64 && io->offset > io_len(io) )
              io_discard_bufstart(io);

Per Hedbor's avatar
Per Hedbor committed
174 175 176 177 178 179 180 181 182
          if( io->len > 64 && ((io->allocated > (io->len)*(1.0+io->max_waste))))
          {
              io->buffer = xrealloc( io->buffer, io->len );
              io->num_malloc++;
              io->allocated = io->len;
          }
      }
  }

183
  static void io_unlink_external_storage( Buffer *io )
184 185
    ATTRIBUTE((noclone,noinline));

186
  static void io_unlink_external_storage( Buffer *io )
187 188
  {
      if( io->sub ) {
189
          io_unlock( get_storage(io->sub,Buffer_program ) );
190 191 192 193 194 195 196 197 198
          free_object( io->sub );
      }
      if( io->source ) free_object( io->source );
      if( io->str ) free_string( io->str );
      io->source = 0;
      io->sub = 0;
      io->str = 0;
  }

199
  PMOD_EXPORT void io_ensure_malloced( Buffer *io, size_t bytes )
Per Hedbor's avatar
Per Hedbor committed
200 201
  {
    if( UNLIKELY(!io->malloced) )
202 203
    {
      /* convert to malloced buffer from a shared one. */
Per Hedbor's avatar
Per Hedbor committed
204
        unsigned char *old = io->buffer;
205

Per Hedbor's avatar
Per Hedbor committed
206
        bytes += io->len;
207

Per Hedbor's avatar
Per Hedbor committed
208
        if (bytes < io->len || bytes + 100 < bytes)
209 210
            Pike_error(msg_out_of_mem_2, bytes + 100);

Per Hedbor's avatar
Per Hedbor committed
211
        bytes += 100;
212

Per Hedbor's avatar
Per Hedbor committed
213 214 215 216 217 218
        io->buffer = xalloc( bytes );
        io->malloced = 1;
        io->allocated = bytes;
        io->num_malloc++;
        memcpy( io->buffer, old, io->len );
        io_unlink_external_storage(io);
219
    }
Per Hedbor's avatar
Per Hedbor committed
220
  }
221

222
  PMOD_EXPORT unsigned char *io_add_space_do_something( Buffer *io, size_t bytes, int force )
223
    ATTRIBUTE((noclone,noinline));
224
  PMOD_EXPORT unsigned char *io_add_space_do_something( Buffer *io, size_t bytes, int force )
Per Hedbor's avatar
Per Hedbor committed
225
  {
226
    if( bytes && io->len+bytes < io->len )
227
      Pike_error("Too large buffer, can not have more than %lu bytes.",
228 229 230
                 (size_t)-1);


Per Hedbor's avatar
Per Hedbor committed
231 232 233 234 235 236 237 238 239 240 241 242
    io_ensure_unlocked(io);
    io_ensure_malloced(io, bytes);

    /*
     * It is actually not certain that checking this is a good idea.
     *
     * The reason being that if the current buffer size is very small
     * (io_len(io)) and the bytes added is large and there is an
     * offset, it makes sense to move now when we do not have to copy
     * as much.
     *
     */
243 244
    if( UNLIKELY((force && io->offset)
              || (io_len(io) && io->offset > io->allocated / 2)) )
245
    {
246 247 248 249 250
      /* more than 50% of the buffer is available before the read pointer,
       * and we can discard that data. Move the data to the beginning, making
       * room for more data.
       */
      io_discard_bufstart(io);
251
    }
Per Hedbor's avatar
Per Hedbor committed
252

Per Hedbor's avatar
Per Hedbor committed
253
    if( UNLIKELY(io->len + bytes > io->allocated) )
254
    {
Per Hedbor's avatar
Per Hedbor committed
255 256 257 258
      /* Actually grow the buffer. */
      size_t growth =
          (io->allocated>>1) +
          (io->allocated>>3);/* io->allocated * 0.625 */
Per Hedbor's avatar
Per Hedbor committed
259

Per Hedbor's avatar
Per Hedbor committed
260 261
      if( growth < bytes )
          growth = bytes + (bytes>>1);
Per Hedbor's avatar
Per Hedbor committed
262

Per Hedbor's avatar
Per Hedbor committed
263 264 265 266
      if( io->allocated + growth < io->allocated )
      {
          growth = bytes+1;
          if( io->allocated + growth < io->allocated )
267
              Pike_error("Overflow in buffer size calculations.\n");
Per Hedbor's avatar
Per Hedbor committed
268 269
      }
      io->buffer = xrealloc( io->buffer, io->allocated + growth );
Per Hedbor's avatar
Per Hedbor committed
270
      io->num_malloc++;
Per Hedbor's avatar
Per Hedbor committed
271
      io->allocated += growth;
272 273 274 275
    }
    return io->buffer+io->len;
  }

Per Hedbor's avatar
Per Hedbor committed
276 277 278 279
  /*! @decl protected bool range_error( int howmuch )
   *!
   *! This function is called when an attempt is made to read out of bounds.
   *!
280
   *! The default implementation simply returns @expr{0@} (zero).
Per Hedbor's avatar
Per Hedbor committed
281
   *!
282 283 284
   *! Override this function to change the behavior.
   *!
   *! @param howmuch
Per Hedbor's avatar
Per Hedbor committed
285 286
   *! The argument @[howmuch] indicates how much data is needed:
   *!
287 288 289
   *! @int
   *!  @value 1..
   *!   Need @[howmuch] bytes more
Per Hedbor's avatar
Per Hedbor committed
290
   *!  @value 0
291 292 293 294 295
   *!   The amount of data needed is not certain.
   *!   This most often happens when  @[sscanf] or @[read_json] is used
   *!  @value ..-1
   *!   Tried to @[unread] -@[howmuch] bytes. There is usually no way to satisfy
   *!   the requested range.
Per Hedbor's avatar
Per Hedbor committed
296
   *!
297 298 299 300
   *!   The only supported way is to extract the data from the buffer,
   *!   add the requested amount of "go backbuffer", add the data
   *!   back, and forward -@[howmuch] bytes.
   *! @endint
Per Hedbor's avatar
Per Hedbor committed
301 302 303 304 305 306 307 308 309
   *!
   *! @returns
   *!
   *! @[true] if the operation should be retried, @[false] otherwise.
   *!
   *! Do not return true unless you have added data to the buffer,
   *! doing so could result in an infinite loop (since no data is
   *! added, the range_error will be called again immediately).
   */
310
  PIKEFUN int(0..1) range_error( int range )
Per Hedbor's avatar
Per Hedbor committed
311
      flags ID_PROTECTED;
312
  {
313
    Pike_sp[-1].u.integer = 0;
Per Hedbor's avatar
Per Hedbor committed
314 315
  }

316
  static void io_range_error_throw( Buffer *io, int howmuch )
Per Hedbor's avatar
Per Hedbor committed
317 318
    ATTRIBUTE((noclone,noinline));

319
  static void io_range_error_throw( Buffer *io, int howmuch )
Per Hedbor's avatar
Per Hedbor committed
320
  {
321 322 323 324 325
    if( io->error_mode )
    {
      struct object *err;
      if( howmuch > 0 )
      {
Per Hedbor's avatar
Per Hedbor committed
326
        push_static_text("Trying to read %d bytes outside allowed range\n");
327 328 329 330
        push_int(howmuch);
        f_sprintf(2);
      }
      else
Per Hedbor's avatar
Per Hedbor committed
331
        push_static_text("Illegal arguments\n");
332 333

      if( io->error_mode != buffer_error_program )
Per Hedbor's avatar
Per Hedbor committed
334
      {
335 336
        ref_push_object( io->this );
        err = clone_object(io->error_mode,2);
Per Hedbor's avatar
Per Hedbor committed
337
      }
338 339 340 341 342 343
      else
        err = clone_object(io->error_mode,1);

      push_object(err);
      f_throw(1);
    }
Per Hedbor's avatar
Per Hedbor committed
344 345
  }

346
  static struct pike_string *io_read_string( Buffer *io, ptrdiff_t len )
Per Hedbor's avatar
Per Hedbor committed
347
    ATTRIBUTE((noclone,noinline));
348
  static size_t io_rewind( Buffer *io, INT_TYPE n );
349

Per Hedbor's avatar
Per Hedbor committed
350 351 352 353 354 355 356
  static void io_do_rewind_on_error( struct rewind_to *e )
  {
    e->io->locked_move--;
    e->io->offset = e->rewind_to;
    free( e );
  }

357
  static void io_rewind_on_error( Buffer *io, ONERROR *x )
Per Hedbor's avatar
Per Hedbor committed
358
  {
359
    struct rewind_to *rew = ALLOC_STRUCT( rewind_to );
360
    io->locked_move++;
Per Hedbor's avatar
Per Hedbor committed
361 362 363 364 365 366 367 368
#if defined(PIKE_DEBUG)
    rew->old_locked_move = io->locked_move;
#endif
    rew->io = io;
    rew->rewind_to = io->offset;
    SET_ONERROR( (*x), io_do_rewind_on_error, rew );
  }

369
  static void io_unset_rewind_on_error( Buffer *io, ONERROR *x )
Per Hedbor's avatar
Per Hedbor committed
370
  {
371
    struct rewind_to *rew = x->arg;
372
#if defined(PIKE_DEBUG)
373
    if( io->locked_move != rew->old_locked_move )
Per Hedbor's avatar
Per Hedbor committed
374
      Pike_fatal( "Invalid io_rewind_on_error nesting\n");
375
#endif
376
    free( rew );
377 378
    UNSET_ONERROR( (*x) );
    io->locked_move--;
Per Hedbor's avatar
Per Hedbor committed
379 380 381 382 383 384 385 386
  }

  static void io_do_unwrite_on_error( struct rewind_to *e )
  {
    e->io->len = e->rewind_to;
    free( e );
  }

387
  static void io_unwrite_on_error( Buffer *io, ONERROR *x )
Per Hedbor's avatar
Per Hedbor committed
388
  {
389
    struct rewind_to *rew = ALLOC_STRUCT( rewind_to );
Per Hedbor's avatar
Per Hedbor committed
390 391 392 393 394
    rew->io = io;
    rew->rewind_to = io->len;
    SET_ONERROR( (*x), io_do_unwrite_on_error, rew );
  }

395
  static void io_unset_unwrite_on_error( Buffer *UNUSED(io), ONERROR *x )
Per Hedbor's avatar
Per Hedbor committed
396 397 398 399 400
  {
    UNSET_ONERROR( (*x) );
    free( x->arg );
  }

401 402
  static ptrdiff_t io_call_write( Buffer *io, struct svalue *fun,
				  ptrdiff_t nbytes )
403 404
    ATTRIBUTE((noclone,noinline));

405
  static void io_set_events( Buffer *io, struct my_file *fd, int extra, int set )
406 407
    ATTRIBUTE((noclone,noinline));

408
  static void io_set_events( Buffer *UNUSED(io), struct my_file *fd, int extra, int set )
409 410 411 412 413 414
  {
    fd->box.revents &= ~((1<<set)|extra);
    if(!SAFE_IS_ZERO(&fd->event_cbs[set]) && fd->box.backend)
      set_fd_callback_events(&fd->box, fd->box.events|(1<<set), 0);
  }

415 416
  static ptrdiff_t io_call_write( Buffer *io, struct svalue *fun,
				  ptrdiff_t bytes )
417 418 419
  {
    if( bytes > 0 )
    {
420
      ptrdiff_t l = 0;
421 422 423 424
      struct pike_string *s;

      io->locked_move++;
      s = io_read_string( io,bytes );
425 426 427 428

      if( s )
      {
        push_string( s );
429 430
        apply_svalue( fun, 1 );
	if (UNLIKELY(TYPEOF(Pike_sp[-1]) != PIKE_T_INT)) {
431
          io->locked_move--;
432 433
	  Pike_error("Invalid return value from write callback.\n");
	}
434 435
        l = Pike_sp[-1].u.integer;
        pop_stack();
436
        if( l < 0 )
437
        {
438
          io->locked_move--;
439 440 441
          io_rewind( io, bytes );
          return -1;
        }
442
        if( bytes > l )
443
        {
444
          io_rewind( io, bytes-l );
445
        }
446
      }
447
      io->locked_move--;
448 449 450 451 452
      return l;
    }
    return -1;
  }

453
  PMOD_EXPORT void io_actually_trigger_output( Buffer *io )
454 455
    ATTRIBUTE((noclone,noinline));

456
  PMOD_EXPORT void io_actually_trigger_output( Buffer *io )
457
  {
458 459 460 461 462 463
    if (UNLIKELY(!io->output.u.object->prog)) {
      /* Destructed object. */
      free_svalue(&io->output);
      SET_SVAL(io->output, PIKE_T_INT, NUMBER_NUMBER, integer, 0);
      return;
    }
464

465
    if ( io->output.u.object ) {
466
      move_svalue(Pike_sp++, &io->output);
467
      SET_SVAL(io->output, PIKE_T_INT, NUMBER_NUMBER, integer, 0);
468 469
      apply_svalue(Pike_sp-1, 0);
      pop_n_elems(2);
470
    }
471 472
  }

473
  static int io_range_error( Buffer *io, ptrdiff_t howmuch )
Per Hedbor's avatar
Per Hedbor committed
474 475
    ATTRIBUTE((noclone,noinline));

476
  static int io_range_error( Buffer *io, ptrdiff_t howmuch )
Per Hedbor's avatar
Per Hedbor committed
477 478 479 480
  {
      int res;
      struct svalue *osp = Pike_sp;

Per Hedbor's avatar
Per Hedbor committed
481
      push_int64( howmuch );
482
      apply_current( f_Buffer_range_error_fun_num, 1 );
Per Hedbor's avatar
Per Hedbor committed
483 484 485 486 487
      res = Pike_sp[-1].u.integer;
      pop_n_elems( Pike_sp-osp );
      if( !res ) io_range_error_throw( io, howmuch );

      return res;
488 489
  }

490
  static int io_avail( Buffer *io, ptrdiff_t len )
491
  {
492 493 494 495 496 497
    if( len < 0 || len + io->offset > io->len )
    {
        if( len < 0 )
            io_range_error_throw( io, 0 );
        else if( io_range_error( io, len+io->len-io->offset ) )
            return io_avail(io,len);
Per Hedbor's avatar
Per Hedbor committed
498
        return 0;
499
    }
500 501 502
    return 1;
  }

503
  static int io_avail_mul( Buffer *io, ptrdiff_t len, ptrdiff_t each  )
Per Hedbor's avatar
Per Hedbor committed
504 505
  {
    /* safely check if len*each is available. */
506 507 508 509 510 511 512 513 514 515 516
    size_t total = io_len(io);
    if( len < 0 || each <= 0 )
    {
      io_range_error_throw( io, 0 );
      return 0;
    }

    if( (total/(size_t)each) < (size_t)len )
    {
        if( io_range_error( io, len+io->len-io->offset ) )
          return io_avail_mul(io,len,each);
Per Hedbor's avatar
Per Hedbor committed
517
        return 0;
518
    }
Per Hedbor's avatar
Per Hedbor committed
519 520 521
    return 1;
  }

522
  static void io_append( Buffer *io, const void *p, size_t bytes )
523 524 525
  {
    memcpy( io_add_space( io, bytes, 0 ), p, bytes );
    io->len += bytes;
526
    io_trigger_output( io );
527 528
  }

529
  static size_t io_read( Buffer *io, void *to, size_t len  )
530 531 532 533 534 535 536 537
  {
    if( !io_avail(io,len))
      return 0;
    memcpy( to, io_read_pointer(io), len );
    io_consume( io, len );
    return len;
  }

538
  static struct pike_string *io_read_string( Buffer *io, ptrdiff_t len )
539 540 541 542 543
  {
    struct pike_string *s;

    if( len > 0x7fffffff )
      Pike_error("This region is too large to convert to a string.\n");
Per Hedbor's avatar
Per Hedbor committed
544

545 546 547
    if( len < 0 )
      return make_shared_binary_string(NULL,0);

Per Hedbor's avatar
Per Hedbor committed
548 549 550
    if( !io_avail(io,len))
     return NULL;

551
    s = begin_shared_string( len );
Per Hedbor's avatar
Per Hedbor committed
552
    io_read( io, s->str, len );
553 554 555
    return end_shared_string(s);
  }

556
  static struct object *io_read_buffer( Buffer *io, size_t len, int do_copy )
557 558
  {
    struct object *b;
559
    Buffer *to;
560 561 562
    if( !io_avail(io,len))
      return NULL;

563
    b = fast_clone_object( Buffer_program );
564
    to = get_storage(b,Buffer_program);
565 566 567 568 569 570

    io_lock( io );

    to->buffer = io_read_pointer(io);
    to->len = len;
    to->sub = Pike_fp->current_object;
571
    add_ref(to->sub);
572 573 574
    io_consume( io, len );

    if( do_copy )
Per Hedbor's avatar
Per Hedbor committed
575
      io_ensure_malloced( to, 0 );
576 577 578 579

    return b;
  }

580
  static int io_read_byte_uc( Buffer *io )
581 582 583 584
  {
    return io->buffer[io->offset++];
  }

585
  static INT_TYPE io_read_number_uc( Buffer *io, size_t len )
586
  {
587
    INT_TYPE res = 0;
Stephen R. van den Berg's avatar
Stephen R. van den Berg committed
588
    while( LIKELY(len--) ) {
Per Hedbor's avatar
Per Hedbor committed
589 590 591 592
      res <<= 8;
      res |= io_read_byte_uc(io);
    }
    return res;
593 594
  }

595 596 597 598 599 600 601 602 603
  static INT_TYPE io_read_le_number_uc( Buffer *io, size_t len )
  {
    INT_TYPE res = 0;
    size_t i;
    for(i=0; i<len; i++)
      res |= (io_read_byte_uc(io) << i*8);
    return res;
  }

604
  static INT_TYPE io_read_signed_number_uc( Buffer *io, size_t len )
605 606
  {
    INT_TYPE res = 0;
Stephen R. van den Berg's avatar
Stephen R. van den Berg committed
607 608 609 610 611 612
    if( LIKELY(len--) ) {
      res = (INT8)io_read_byte_uc(io);
      while( LIKELY(len--) ) {
        res <<= 8;
        res |= io_read_byte_uc(io);
      }
613 614 615 616
    }
    return res;
  }

617
  static INT64 io_read_number( Buffer *io, size_t len, int endian )
618
  {
619
    INT64 res;
620 621
    if( !io_avail(io, len) )
      return -1;
Per Hedbor's avatar
Per Hedbor committed
622
    /* ensure only leading 0:s */
623 624 625 626
    for (; UNLIKELY(len > SIZEOF_INT_TYPE); len--)
      if( UNLIKELY(io_read_byte_uc(io)) )
        Pike_error("Integer (%dbit) overflow.\n", SIZEOF_INT_TYPE*8);

627 628 629
    /* NB: endian == 0 for little endian,
     *               1 for big endian.
     */
630 631 632 633
    if( endian )
      res = io_read_number_uc( io, len );
    else
      res = io_read_le_number_uc( io, len );
634 635 636 637

    if ( UNLIKELY(res < 0) )
      Pike_error("Signed (%dbit) overflow.\n", SIZEOF_INT_TYPE*8);
    return res;
638 639
  }

640
  static struct object *io_read_bignum( Buffer *io, size_t len, int endian )
641
  {
642 643 644 645 646
    struct object *o;
    MP_INT *i;
    unsigned char *p;

    if( !io_avail(io,len) ) return NULL;
647
    o = fast_clone_object(bignum_program);
648
    i = (void*)o->storage;
649 650 651 652 653
    /* NB: endian == -1 for little endian,
     *                0 for native,
     *                1 for big endian.
     */
    mpz_import( i, len, endian, 1, endian, 0, io_read_pointer(io) );
654 655
    io_consume(io,len);
    return o;
656 657
  }

658
  static void io_add_bignum( Buffer *io, struct object *o, int width )
659 660
    ATTRIBUTE((noclone,noinline));

661
  static void io_add_bignum( Buffer *io, struct object *o, int width )
662
  {
663 664 665
    MP_INT *i = (void*)o->storage;
    MP_INT tmp;
    int free;
666 667
    unsigned char*d;
    struct pike_string *s;
668
    int pad = 0;
669
    ptrdiff_t bytes;
670
    size_t exp;
Per Hedbor's avatar
Per Hedbor committed
671

672 673 674 675 676 677 678
    if( mpz_sgn( i ) < 0 )
    {
      mpz_init( &tmp );
      mpz_add_ui( &tmp, i, 1);
      pad = 0xff;
      i = &tmp;
    }
Per Hedbor's avatar
Per Hedbor committed
679

680
    bytes = (mpz_sizeinbase( i, 2 )+7) / 8;
Per Hedbor's avatar
Per Hedbor committed
681

682
    if( bytes > width )
683
      Pike_error("Number too large to store in %d bits.\n", width*8);
Per Hedbor's avatar
Per Hedbor committed
684

685 686 687
    d = io_add_space( io, width, 0 );
    io->len += width;

688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
    if( width > bytes )
    {
      memset( d, pad, width-bytes );
      d += width-bytes;
    }

    mpz_export( d, &exp, 1, 1, 1, 0, i );
    if( !exp )
    {
      /* if i is 0 mpz_export will not write anything.
         Handle that by zeroing the byte that should have been written.
      */
#ifdef PIKE_DEBUG
      if( bytes != 1 )
        Pike_fatal("Oddities abound\n");
#endif
      *d = pad;
    }
    if( pad )
Per Hedbor's avatar
Per Hedbor committed
707
    {
708 709 710
      while(exp--)
        *d++ ^= 0xff; /* pad, but that is 0xff */
      mpz_clear(&tmp);
Per Hedbor's avatar
Per Hedbor committed
711
    }
712
    io_trigger_output( io );
Per Hedbor's avatar
Per Hedbor committed
713 714
  }

715
  static void io_add_int_uc( Buffer *io, ptrdiff_t i, size_t bytes )
Per Hedbor's avatar
Per Hedbor committed
716
  {
Per Hedbor's avatar
Per Hedbor committed
717
    unsigned char *x = io->buffer+io->len;
718 719 720
    io->len += bytes;
    while(bytes--)
    {
721
      x[bytes] = i;
722 723
      i>>=8;
    }
Per Hedbor's avatar
Per Hedbor committed
724 725
  }

726
  static size_t io_add_int( Buffer *io, ptrdiff_t i, size_t bytes )
Per Hedbor's avatar
Per Hedbor committed
727
  {
728
    io_add_space(io, bytes, 0);
Per Hedbor's avatar
Per Hedbor committed
729
    io_add_int_uc( io, i, bytes );
730
    io_trigger_output( io );
731 732 733
    return io_len( io );
  }

734
  static size_t io_rewind( Buffer *io, INT_TYPE n )
735
  {
736 737 738 739 740 741 742 743
    if( n < 0 || (io->offset < (unsigned)n) )
    {
        if( n < 0 )
            io_range_error_throw( io, 0 );
        else if( io_range_error(io,-(long)(n-io->offset)) )
          return io_rewind( io, n );
      return -1;
    }
744
    io->offset -= n;
745
    io_trigger_output( io );
746 747 748
    return io->offset;
  }

749
  static void io_append_byte_uc( Buffer *io, unsigned char byte )
750 751 752 753
  {
      io->buffer[io->len++] = byte;
  }

754
  static void io_append_short_uc( Buffer *io, unsigned short shrt )
755
  {
756
    set_unaligned16( io->buffer+io->len, htons(shrt));
757
    io->len+=2;
758 759
  }

760
  static void io_append_int_uc( Buffer *io, unsigned INT32 i )
761
  {
762
    set_unaligned32( io->buffer+io->len, htonl(i));
763
    io->len+=4;
764 765
  }

Per Hedbor's avatar
Per Hedbor committed
766

767
  static size_t io_svalue_len( Buffer *io, struct svalue *p )
Per Hedbor's avatar
Per Hedbor committed
768
  {
769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
    switch( TYPEOF(*p) ) {
      case PIKE_T_INT:
        return 1;
      case PIKE_T_STRING:
        if( !p->u.string->size_shift )
          return p->u.string->len;
        break;
      case PIKE_T_ARRAY: {
        size_t len;
        struct array *argp = p->u.array;
        INT_TYPE i;
        DECLARE_CYCLIC();

        if (BEGIN_CYCLIC(io, argp))
          Pike_error("Attempt to append a cyclic array to a buffer.\n");

        for(len=i=0; i<argp->size; i++ )
          len += io_svalue_len( io, argp->item+i );

        END_CYCLIC();
789
        return len;
790 791 792 793 794 795 796
      }
      case PIKE_T_OBJECT: {
        size_t len;
        if( get_memory_object_memory( p->u.object, NULL, &len, NULL ) )
          return len;
        break;
      }
797
    }
798
    Pike_error("Illegal argument (not an 8bit string or 8bit buffer object).\n");
Per Hedbor's avatar
Per Hedbor committed
799 800 801
  }

  /* NOTE: Can return negative integers. */
802 803 804
  static INT_TYPE get_small_int( struct svalue *s )
    ATTRIBUTE((noclone,noinline));

805 806
  static INT_TYPE get_small_int( struct svalue *s )
  {
807 808 809 810 811 812 813 814
    if( LIKELY(TYPEOF(*s) == PIKE_T_INT) )
      return s->u.integer;

    if( is_bignum_object_in_svalue( s ) )
    {
      INT64 i64;
      if( int64_from_bignum( &i64, s->u.object ) )
        return i64;
815
      Pike_error("Too big bignum, can not fit in indicated width.\n");
816
    }
817
    Pike_error("Non integer argument.\n");
818 819
  }

820
  static void io_append_svalue( Buffer *io, struct svalue *p )
821 822
    ATTRIBUTE((noinline));

823
  static void io_append_svalue( Buffer *io, struct svalue *p )
Per Hedbor's avatar
Per Hedbor committed
824 825 826 827 828 829
  {
    switch( TYPEOF(*p) )
    {
      case PIKE_T_STRING:
        {
          struct pike_string *s = p->u.string;
830
	  if( !s->len ) return;
831
          if( s->size_shift ) Pike_error("Buffer only handles 8bit data.\n");
Per Hedbor's avatar
Per Hedbor committed
832 833
          if( !io->buffer )
          {
834
#ifdef PIKE_DEBUG
835
	    if (io->str) Pike_fatal("Buffer with string but NULL buffer.\n");
836
#endif
Per Hedbor's avatar
Per Hedbor committed
837 838 839
            io->str = s;
            io->buffer = (unsigned char*)s->str;
            io->len = s->len;
840
            add_ref(s);
841
            io_trigger_output( io );
Per Hedbor's avatar
Per Hedbor committed
842 843 844 845 846 847 848 849 850
          }
          else
            io_append( io, s->str, s->len );
        }
        break;
      case PIKE_T_ARRAY:
        {
          struct array *argp = p->u.array;
          INT_TYPE i;
851 852 853 854 855
	  DECLARE_CYCLIC();

	  if (BEGIN_CYCLIC(io, argp))
	    Pike_error("Attempt to append a cyclic array to a buffer.\n");

Per Hedbor's avatar
Per Hedbor committed
856 857
          for(i=0; i<argp->size; i++ )
            io_append_svalue( io, argp->item+i );
858 859

	  END_CYCLIC();
Per Hedbor's avatar
Per Hedbor committed
860 861 862 863
        }
        break;
      case PIKE_T_OBJECT:
        {
864 865 866 867 868 869
          size_t len;
          void *ptr;
          struct sysmem *s;
          enum memobj_type t = get_memory_object_memory( p->u.object, &ptr, &len, NULL );

          if( !io->buffer && t==MEMOBJ_SYSTEM_MEMORY )
Per Hedbor's avatar
Per Hedbor committed
870
          {
871 872 873 874
            io->buffer = ptr;
            io->len = len;

            io->source = p->u.object;
875
            add_ref(io->source);
876
            return;
Per Hedbor's avatar
Per Hedbor committed
877
          }
878 879
          if( t != MEMOBJ_NONE )
            io_append( io, ptr, len );
Per Hedbor's avatar
Per Hedbor committed
880
          else
881
            Pike_error("Unsupported argument type.\n");
Per Hedbor's avatar
Per Hedbor committed
882 883 884 885 886 887 888 889 890
        }
        break;
      case PIKE_T_INT:
        {
          unsigned char a = p->u.integer;
          io_append( io, &a, 1 );
        }
    }
  }
891

Per Hedbor's avatar
Per Hedbor committed
892
#undef THIS
893
#define THIS (&(((struct Buffer_struct *)Pike_fp->current_storage)->b))
894 895


896 897
  /* pike functions */

898
  /*! @decl int(-1..) input_from( Stdio.Stream f, int|void nbytes )
899
   *!
900 901
   *! Read data from @[f] into this buffer. If @[nbytes] is not
   *! specified, read until there is no more data to read (currently).
Per Hedbor's avatar
Per Hedbor committed
902
   *!
903 904
   *! Returns the amount of data that was read, or @expr{-1@} on
   *! read error.
Per Hedbor's avatar
Per Hedbor committed
905 906 907 908
   *!
   *! @note
   *! Please note that this funcition will read all data from the
   *! filedescriptor unless it's set to be non-blocking.
909
   */
910
  PIKEFUN int(-1..) input_from( object f, int|void _nbytes, int|void _once )
911
  {
912
    Buffer *io = THIS;
913
    size_t sz = io_len( io );
914
    size_t bread = 0, nbytes = (size_t)-1;
915
    struct my_file *fd;
916
    int once = 0;
917

918
    if( _nbytes ) {
919
        nbytes = _nbytes->u.integer;
920 921
	if (!nbytes) RETURN 0;
    }
922

923 924 925
    if( _once )
        once = _once->u.integer;

926 927 928 929
    if( (fd = get_storage( f, file_program )) )
    {
      while( 1 )
      {
930
        unsigned char *ptr = io_add_space( io, READ_CHUNKSIZE, 0 );
931 932
        int res;

933
        res = fd_read( fd->box.fd, ptr, MINIMUM(READ_CHUNKSIZE, nbytes) );
934 935 936 937 938 939

        if( res == -1 && errno == EINTR )
          continue;

        if( res <= 0 )
          break;
940

941
        nbytes -= res;
942 943
        io->len += res;
        bread += res;
944
        if( res != READ_CHUNKSIZE || once || !nbytes )
Per Hedbor's avatar