Download Reference Manual
The Developer's Library for D
About Wiki Forums Source Search Contact

BufferedInput stress test

Moderators: kris

Posted: 12/14/09 16:42:28

I'm using my own version of BufferedInput?, because I need getByte/lookByte functionality that BufferedInput? is missing. To test if everything is correct I've developed a little stress test. What it does is write some data to two temp files. One will be used naked and one trough BufferedInput? (or my own version of it). Then do a lot of random seeking and reading from the two sources and compare the output. I tried the same test with Tango's BufferedInput? and it fails. Forgive me if it is due to my lack of understanding how it works... Here is the test:

import tango.math.random.Kiss;
import tango.io.device.TempFile;
import tango.io.stream.Buffered;
import tango.io.Stdout;

alias Stdout trace;

class Test {
	this() {
		if(t is null) {
			t = new TempFile;
			t2 = new TempFile;
			for(char i = 0; i<255; ++i) {
				auto b = cast(void[])[i];
				t.write(b);
				t2.write(b);
			}
		}		
		t.seek(0);
		t2.seek(0);
		s = new BufferedInput(t2,32);
	}

	bool once(uint what,uint n,bool trc = false) {
		if(what == 0) {
			if(trc) trace("seek",n).newline.flush;
			t.seek(n,IOStream.Anchor.Begin);
			s.seek(n,IOStream.Anchor.Begin);
		}
		else if(what == 1) {
			if(trc) trace("read",n).newline.flush;
			auto b = new char[n];
			auto b2 = new char[n];
			auto p1 = t.position;
			auto p2 = s.position;
			auto r = t.read(b);
			auto r2 = s.read(b2);
			if(r == s.Eof || (b[0..r<r2?r:r2] == b2[0..r<r2?r:r2])) return true;
			else {
				trace("Bad",r,r2,"t pos",p1,"s pos",p2,"\nt buf",b[0..r<r2?r:r2],"\ns buf",b2[0..r<r2?r:r2]).newline.flush;
				return false;
			}
		}
		return true;
	}

	bool many(uint[2][] s,bool trc = false) {
		foreach(i;s) if(!once(i[0],i[1],trc)) return false;
		return true;
	}

	static TempFile t,t2;
	BufferedInput s;
}

int main(char[][] argv) {
	//*
	//if 1000 tests are ok, make it 1000000 to be sure :)
	for(uint c = 0; c<1_000; ++c) {
		//trace("-----------------------------",c).newline.flush;
		scope t = new Test;
		uint[2][5] m;
		for(uint i = 0; i<5; ++i) {
			m[i][] = [Kiss.instance.natural(2),Kiss.instance.natural(256)];
		}
		if(!t.many(m)) trace(m).newline.flush;
	}
	return 0;
	//*/
	//to single test with particular data. the data will be printed when a test fails
	//auto t = new Test;
	//t.many([[0, 155], [0, 0], [1, 17], [0, 186], [1, 50]],true);
	//return 0;
}

By the way, it would be great if BufferedInput? and device.Array had getByte/lookByte - optimized version of read, to read a single byte. lookByte would return the first byte in the buffer, but not eat it. Useful for parsing streams. Or does Tango have similar functionality? I usually develop something and then find out that Tango already has that, but I didn't know its name...

Author Message

Posted: 12/14/09 17:40:01

One more thing. BufferedInput?.read should read from the underlying stream in the cases where (readable < dst.length), not just read available data from the buffer and make the user call read again. This is the reason the length of the data read from the two streams above may be different. IMO this is incorrect.

Posted: 12/14/09 17:41:31

Sharing my own implementation:

/// this class provides a way to read the data from the underlying stream byte-by-byte, but provides some caching, so read() is not invoked for each byte

module flowerd.io.ByteStream;

import tango.io.model.IConduit;

class ByteInput : InputStream {
	const defaultChunkSize = 256;

	this(InputStream input,uint chunksize = defaultChunkSize) {
		this.chunksize = chunksize;
		this._input = input;
		this.buffer.length = chunksize;
	}

	long position() {
		return bufPos + bufPtr;
	}

	long seek(long pos,Anchor anchor) {
		//todo: this could be optimize more if input size is known
		if(anchor == Anchor.Begin && pos >= bufPos && pos < bufPos + bufEnd) {
			bufPtr = pos - bufPos;
			return pos;
		}
		auto r = _input.seek(pos,anchor);
		bufPtr = bufEnd = 0;
		if(r >= 0) bufPos = r;
		return r;
	}

	int getByte() {
		return (bufPtr >= bufEnd && !fillBuffer()) ? Eof : buffer[bufPtr++];
	}

	int lookByte() {
		return (bufPtr >= bufEnd && !fillBuffer()) ? Eof : buffer[bufPtr];
	}

	uint read(void[] dst) {
		if(dst is null) return 0;
		if(bufPtr >= bufEnd) {
			if(dst.length >= buffer.length) {
				auto r = _input.read(dst);
				if(r != _input.Eof) {
					bufPos += bufEnd + r;
					bufPtr = bufEnd = 0;
				}
				return r;
			}
			else {
				if(!fillBuffer()) return Eof;
			}

			auto bufLen = bufEnd - bufPtr;
			if(dst.length <= bufLen) {
				dst[] = buffer[bufPtr..bufPtr+dst.length];
				bufPtr += dst.length;
				return dst.length;
			}
			else {
				dst[0..bufLen] = buffer[bufPtr..bufPtr+bufLen];
				bufPtr += bufLen;
				return bufLen;
			}
		}
		else {
			auto bufLen = bufEnd - bufPtr;
			if(dst.length <= bufLen) {
				dst[] = buffer[bufPtr..bufPtr+dst.length];
				bufPtr += dst.length;
				return dst.length;
			}
			else {
				dst[0..bufLen] = buffer[bufPtr..bufEnd];
				auto r = _input.read(dst[bufLen..$]);
				if(r == _input.Eof) {
					bufPtr = bufEnd;
					return bufLen;
				}
				else {
					bufPos += bufEnd + r;
					bufPtr = bufEnd = 0;
					return r + bufLen;
				}
			}
		}
	}

	//todo: not tested
	void[] load(size_t max = -1) {
		throw new Exception("not implemented");
		/*if(bufPtr >= bufEnd) {
			auto r = input.load(dst);
			if(r != Eof) bufPos += r;
			return r;
		}
		else {
			auto bufLen = bufEnd - bufPtr;
			if(dst.length < bufLen) dst.length = bufLen;
			dst[0..bufLen] = buffer[bufPtr..bufEnd];
			bufPtr = bufEnd;

			auto dst2 = input.load();
			auto finLen = bufLen + dst2.length;
			if(dst2.length) {
				if(dst.length < finLen) dst.length = finLen;
				dst[bufLen..finLen] = dst2[];
			}

			return dst[0..finLen];
		}*/
	}

	IOStream flush() {
		bufPos = bufPtr = bufEnd = 0;
		return _input.flush();
	}

	void close() {
		_input.close();
	}

	IConduit conduit() {
		return _input.conduit();
	}

	InputStream input() {
		return _input;
	}

protected:
	bool fillBuffer() {
		bufEnd = _input.read(buffer); //do this first to avoid messing variables if _input.read throws
		bufPtr = 0;
		bufPos += bufEnd;
		if(bufEnd == _input.Eof) bufEnd = 0;
		return bufPtr < bufEnd;
	}

	uint bufPtr,bufEnd;
	long bufPos;
	char[] buffer;
	InputStream _input;
	uint chunksize;
}

Posted: 12/15/09 20:02:52

InputBuffered?.slice(count, bool) does not eat the input when the second argument is set false