www.digitalmars.com         C & C++   DMDScript  

digitalmars.D - My stream concept

reply Regan Heath <regan netwin.co.nz> writes:
------------HThpJ5CpXXRdSLQVK4P5jO
Content-Type: text/plain; format=flowed; charset=iso-8859-15
Content-Transfer-Encoding: 8bit

Typical stream implementations I have seen use the following heirarchy:

class Stream {
}
class FileStream : Stream {
}
class SocketStream : Stream {
}

in other words you have to design/derive a class from Stream for each type 
of Stream you want to implement.

I always thought this was kinda backwards, and limited, in fact after 
Arcane Jill said something along the lines of "anything that reads and 
writes bytes _is_ a stream" I got to thinking surely I can use templates 
to make that statement true.


The way I have done it is to define some interfaces:

interface InputStream
{
	ulong read(void* buffer, ulong length);
	ulong available();
	bool eof();
}

interface OutputStream
{
	ulong write(void* buffer, ulong length);
}

and a Stream and BufferedStream template class. Any object that supports 
those interfaces (as does my RawFile class) can then be extended by the 
two template classes to give full stream functionality by some simple 
aliases, example:

alias Stream!(RawFile) FileStream;
alias BufferedStream!(FileStream,2048) File;

I have attached the code, it is far from complete and probably buggy as 
heck as I have not done any testing (or unittests - I know I'm bad) this 
was simply proof of concept.

Any and all comments are welcome/desired.

Regan

-- 
Using M2, Opera's revolutionary e-mail client: http://www.opera.com/m2/
------------HThpJ5CpXXRdSLQVK4P5jO
Content-Disposition: attachment; filename=syserror.d
Content-Type: application/octet-stream; name=syserror.d
Content-Transfer-Encoding: 8bit

module lib.syserror;

private import std.string;
private import std.c.windows.windows;
private import std.c.stdarg;

extern (Windows) {
	DWORD FormatMessageA(
		DWORD dwFlags,
		LPCVOID lpSource,
		DWORD dwMessageId,
		DWORD dwLanguageId,
		LPTSTR lpBuffer,
		DWORD nSize,
		va_list *Arguments
	);

	static uint FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100;
	static uint FORMAT_MESSAGE_IGNORE_INSERTS  = 0x00000200;
	static uint FORMAT_MESSAGE_FROM_STRING     = 0x00000400;
	static uint FORMAT_MESSAGE_FROM_HMODULE    = 0x00000800;
	static uint FORMAT_MESSAGE_FROM_SYSTEM     = 0x00001000;
	static uint FORMAT_MESSAGE_ARGUMENT_ARRAY  = 0x00002000;
	static uint FORMAT_MESSAGE_MAX_WIDTH_MASK  = 0x000000FF;

	WORD MAKELANGID(WORD p, WORD s)  { return (((cast(WORD)s) << 10) |
cast(WORD)p); }
	WORD PRIMARYLANGID(WORD lgid)    { return (cast(WORD)lgid & 0x3ff); }
	WORD SUBLANGID(WORD lgid)        { return (cast(WORD)lgid >> 10); }

	static uint LANG_NEUTRAL = 0x00;
	static uint SUBLANG_DEFAULT = 0x01;

	alias HGLOBAL HLOCAL;

	HLOCAL LocalFree(HLOCAL hMem);
}

extern (C) char *strerror(int);

class SysError
{
	static char[] msg(uint errcode)
	{
		char[] text;

		version(Windows)
		{
			LPVOID lpMsgBuf;
			DWORD r;

			r = FormatMessageA( 
				FORMAT_MESSAGE_ALLOCATE_BUFFER | 
				FORMAT_MESSAGE_FROM_SYSTEM | 
				FORMAT_MESSAGE_IGNORE_INSERTS,
				null,
				errcode,
				MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language
				cast(LPTSTR)&lpMsgBuf,
				0,
				null 
			);
			/* Remove \r\n from error string */
			if (r >= 2) r-= 2;
			text = (cast(char *)lpMsgBuf)[0..r];
			LocalFree(cast(HLOCAL)lpMsgBuf);
		}
		else
		{
			char* pemsg;
			uint r;

			pemsg = strerror(errcode);
			r = strlen(pemsg);
			/* Remove \r\n from error string */
			if (pemsg[r-1] == '\n') r--;
			if (pemsg[r-1] == '\r') r--;
			text = pemsg[0..r];
		}
		
		return text;
	}
}

------------HThpJ5CpXXRdSLQVK4P5jO
Content-Disposition: attachment; filename=file.d
Content-Type: application/octet-stream; name=file.d
Content-Transfer-Encoding: 8bit

module lib.file;

import std.c.windows.windows;
import std.file;

import lib.stream;
import lib.syserror;

extern (Windows)
{
	export {
		BOOL PeekNamedPipe(
			HANDLE hNamedPipe,
			LPVOID lpBuffer,
			DWORD nBufferSize,
			LPDWORD lpBytesRead,
			LPDWORD lpTotalBytesAvail,
			LPDWORD lpBytesLeftThisMessage
		);
	}
}

enum FileMode {
	NONE	= 0x00,
	READ	= 0x01,
	CREATE	= 0x02,
	APPEND	= 0x04,
	WRITE	= 0x08,
	NEW		= 0x10
}

private enum FileFlag {
	NONE	= 0x0,
	EOF		= 0x1
}

class FileException : Exception
{
	char[] message;
	uint code;

	this(char[] _message, DWORD _code)
	{
		super(SysError.msg(_code));
		message = _message;
		code = _code;		
	}
	char[] toString()
	{
		return message ~ ", " ~ super.toString();
	}
	void print()
	{
		printf("%.*s\n",toString());
	}
}

class RawFile : InputStream, OutputStream, Seekable
{
	this(char[] file = null, FileMode mode = FileMode.NONE)
	{
		open(file,mode);
	}

	~this()
	{
		close();
	}

	void open(char[] file = null, FileMode mode = FileMode.NONE)
	{
		DWORD access,flags,share;

		if (file is null) return ;

		close();	

		if (mode & FileMode.READ)	{ access |= GENERIC_READ;  flags |= OPEN_EXISTING; }
		if (mode & FileMode.CREATE) { access |= GENERIC_WRITE; flags |=
CREATE_ALWAYS; }
		if (mode & FileMode.APPEND) { access |= GENERIC_WRITE; flags |= OPEN_ALWAYS; }
		if (mode & FileMode.WRITE)  { access |= GENERIC_WRITE; }
		if (mode & FileMode.NEW)    { access |= GENERIC_WRITE; flags |= CREATE_NEW; }

		h = CreateFileA(
			std.file.toMBSz(file)
			,access					/* GENERIC_READ|GENERIC_WRITE */
			,share					/* FILE_SHARE_DELETE|FILE_SHARE_READ|FILE_SHARE_WRITE */
			,null					/* SECURITY_ATRIBUTES */
			,flags					/* CREATE_NEW|CREATE_ALWAYS|OPEN_EXISTING|OPEN_ALWA
S|TRUNCATE_EXISTING */
			,0						/* FILE_ATTRIBUTE_ARCHIVE|FILE_ATTRIBUTE_HIDDEN|FILE_ATTRIBUTE_NORMAL|
									FILE_ATTRIBUTE_OFFLINE|FILE_ATTRIBUTE_READONLY|FILE_ATTRIBUTE_SYSTEM|
									FILE_ATTRIBUTE_TEMPORARY *//*
									FILE_FLAG_WRITE_THROUGH|FILE_FLAG_OVERLAPPED|FILE_FLAG_NO_BUFFERING|FILE_FLAG_RANDOM_ACCESS|
									FILE_FLAG_SEQUENTIAL_SCAN|FILE_FLAG_DELETE_ON_CLOSE|FILE_FLAG_BACKUP_SEMANTICS
									FILE_FLAG_POSIX_SEMANTICS|FILE_FLAG_OPEN_REPARSE_POINT|FILE
FLAG_OPEN_NO_RECALL */
									/* SECURITY_ANONYMOUS|SECURITY_IDENTIFICATION|SECURITY_IMPERSONATION|SECURITY_DELEGATION
									SECURITY_CONTEXT_TRACKING|SECURITY_EFFECTIVE_ONLY */
			,null);					/*handle: copy attributes of this file*/

		if (h == INVALID_HANDLE_VALUE)
			throw new FileException("Failed to open ("~file~")",GetLastError());

		return ;
	}

	void close()
	{
		if (h == INVALID_HANDLE_VALUE) return ;
		CloseHandle(h);
	}
	
	ulong read(void* buffer, ulong length)
	{
		DWORD bytes;

		if (ReadFile(h,buffer,length,&bytes,null) == 0)
			throw new FileException("read failed",GetLastError());
		
		if (bytes != length)			
			flags |= FileFlag.EOF;

		return bytes;
	}

	ulong available()
	{
		DWORD szLow,szHigh;
		LONG cLow,cHigh;
		ulong size,cur;

		cLow = SetFilePointer(h,0,&cHigh,cast(LONG)SeekPos.CURRENT);
		if (cLow == 0xFFFFFFFF && GetLastError() != 0)
			throw new FileException("available seek failed",GetLastError());

		szLow = GetFileSize(h,&szHigh);
		if (szLow == 0xFFFFFFFF && GetLastError() != 0)
			throw new FileException("available getsize failed",GetLastError());

		size = szLow|(szHigh<<32);
		cur = cLow|(cHigh<32);

		if (cur > size)
			return 0;

		return size-cur;
	}

	bool eof()
	{
		return (flags & FileFlag.EOF) != 0;
	}

	ulong write(void *buffer, ulong length)
	{
		DWORD bytes;

		if (WriteFile(h,buffer,length,&bytes,null) == 0)
			throw new FileException("write failed",GetLastError());

		return bytes;
	}

	ulong seek(long bytes, SeekPos origin)
	{
		LONG dLow,dHigh;

		splitLong(bytes,dLow,dHigh);
		
		dLow = SetFilePointer(h,dLow,&dHigh,origin);

		if (dLow == 0xFFFFFFFF && GetLastError() != 0)
			throw new FileException("seek failed",GetLastError());

		return combineInt(dLow,dHigh);
	}

	ulong seekBack(long bytes)
	{
		return seek(-bytes,SeekPos.CURRENT);
	}

	ulong seekFwd(long bytes)
	{
		return seek(bytes,SeekPos.CURRENT);
	}

	ulong seekStart()
	{
		return seek(0,SeekPos.BEGIN);
	}

	ulong seekEnd()
	{
		return seek(0,SeekPos.END);
	}

private:
	HANDLE h = INVALID_HANDLE_VALUE;
	FileFlag flags;

	ulong combineInt(LONG low, LONG high)
	{
		return low|(high<<32);
	}

	void splitLong(ulong x, LONG low, LONG high)
	{
		low  = (x&0xFFFFFFFF);
		high = (x>>32);
	}
}

alias Stream!(RawFile)					FileStream;
alias BufferedStream!(FileStream,2048)	File;

/*
"r"  - READ        - read, fails if file does not exist.
"w"  - CREATE      - write, overwrite existing.
"a"  - APPEND      - write, create if not exist.
"r+" - READ|WRITE  - read, write, fails if file does not exist.
"w+" - READ|CREATE - read, write, overwrite existing.
"a+" - READ|APPEND - read, append, create if not exist.
""   - WRITE|NEW   - write, fail if file exist.
*/

/*
WUC        - write, update, create
RWUC       - read, write, update, create
WAUC  "a"  - write, append, update, create
RWAUC "a+" - read, write, append, update, create
WOC   "w"  - write, overwrite, create
RWOC  "w+" - read, write, overwrite, create
RC         - read, create
RWC        - read, write, create
RU    "r"  - read, existing
WU         - write, existing
RWU   "r+" - read, write, existing
RAU        - read, append, existing
WAU        - write, append, existing
RWAU       - read, write, append, existing
WO         - write, overwrite
RWO        - read, write, overwrite
*/

/*
The open() routine seems to controls four aspects:
 
  Access: Read, Write, Both, Neither
  File-Exists action: Use, Overwrite, Fail
  File-Not-Exists action: Create, Fail
  Initial Seek: Start-of-File (normal), End-of-File (append)
 
This leads to 48 different combinations, some of which are not useful (eg.
neither read nor write, fails if exists and if not-exists, read starting
from end of file, etc...). So after removing the useless combinations, we
are left with 20 possible ones.
 
I've devised six codes that encompass these combinations:
R : read access from start of file
W : write access from start of file
A : write access from end of file
U : use existing file
O : overwrite existing file
C : create non-existing file
 
So the useful combinations that either open a file or fail are (I've also
show Regan's equivalent codes) ...
 
RWUC      - read, write, use existing, create if not existing.
RAUC "a+" - read, append (to existing), create if not exist.
RWOC "w+" - read, write, overwrite existing(, create if not existing).
RWU  "r+" - read, write, fails if file does not exist.
RAU       - read, append to existing, fail if not exist.
RWC       - read, write, fail if exists, create if not existing.
RWO       - read, write, overwrite if exists, fail if not existing.
RAO       - read, append if exists, fail if not existing.
WUC       - write, use if exists, create if not existing.
AUC  "a"  - write(, append if exists), create if not exist.
WOC  "w"  - write, overwrite existing(, create if not existing).
RUC       - read, use if exists, create if not existing.
ROC       - read, overwrite if exists, create if not existing. (Used to
                 always create an empty file).
RU   "r"  - read, fails if file does not exist.
RC        - read, fail if exists, create if not existing. (Used to create
                 an empty file if it doesn't already exist).
RO        - read, overwrite if exists, fail if not existing. (Used to wipe
                 an existing file).
WU        - write, use if file exists, fail if not existing.
AU        - write, append if file exists, fail if not existing.
WC    ""  - write, fail if file exists(, create if not existing).
WO        - write, overwrite if exists, fail if not existing.(Used to
                 replace an existing file).
*/

------------HThpJ5CpXXRdSLQVK4P5jO
Content-Disposition: attachment; filename=stream.d
Content-Type: application/octet-stream; name=stream.d
Content-Transfer-Encoding: 8bit

module lib.stream;

version (Windows) {
	static char[] EOL = "\r\n";
}
else {
	static char[] EOL = "\n";
}

interface InputStream
{
	ulong read(void* buffer, ulong length);
	ulong available();
	bool eof();
}

interface OutputStream
{
	ulong write(void* buffer, ulong length);
}

enum SeekPos
{
	BEGIN,
	CURRENT,
	END
}

interface Seekable
{
	ulong seek(long bytes, SeekPos origin);
	ulong seekBack(long bytes);
	ulong seekFwd(long bytes);
	ulong seekStart();
	ulong seekEnd();
}

class Stream(alias T) : T
{
	this()
	{
	}

	alias T.read read;

	bool read(out byte x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out ubyte x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out short x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out ushort x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return
true; return false; }
	bool read(out int x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out uint x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out long x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out ulong x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out float x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out double x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return
true; return false; }
	bool read(out real x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out ireal x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out creal x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out char x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }
	bool read(out wchar x)	{ if (super.read(&x,x.sizeof) == x.sizeof) return true;
return false; }

	bool read(out char[] x)	{
		uint len;

		if (!this.read(len))
			return false;
		
		x.length = len;
		if (!super.read(x,len*char.sizeof))
			return false;

		return true;
	}

	bool read(out wchar[] x)	{
		uint len;

		if (!this.read(len))
			return false;
		
		x.length = len;
		if (!super.read(x,len*wchar.sizeof))
			return false;

		return true;
	}

	bool read(out dchar[] x)	{
		uint len;

		if (!this.read(len))
			return false;

		x.length = len;
		if (!super.read(x,len*dchar.sizeof))
			return false;

		return true;
	}

	bool readLine(inout char[] line)
	{
		uint i = 0;

		if (super.eof())
			return false;

		if (line.length == 0)
			line.length = 128;

		while(true) {	
			foreach(inout char c; line[i..line.length]) {
				if (!this.read(c)) break;
				i++;
				if (c == '\r') this.read(c);
				if (c == '\n') break;
			}
			if (i == 0) return false;
			if (i != line.length) {
				if (line[i-1] == '\n') {
					line.length = i-1;
					break;
				}
			}
			line.length = line.length*2;
		}

		return true;
	}

	alias T.write write;

	void write(byte x)		{ super.write(&x,x.sizeof); }
	void write(ubyte x)		{ super.write(&x,x.sizeof); }
	void write(short x)		{ super.write(&x,x.sizeof); }
	void write(ushort x)	{ super.write(&x,x.sizeof); }
	void write(int x)		{ super.write(&x,x.sizeof); }
	void write(uint x)		{ super.write(&x,x.sizeof); }
	void write(long x)		{ super.write(&x,x.sizeof); }
	void write(ulong x)		{ super.write(&x,x.sizeof); }
	void write(float x)		{ super.write(&x,x.sizeof); }
	void write(double x)	{ super.write(&x,x.sizeof); }
	void write(real x)		{ super.write(&x,x.sizeof); }
	void write(ireal x)		{ super.write(&x,x.sizeof); }
	void write(creal x)		{ super.write(&x,x.sizeof); }
	void write(char x)		{ super.write(&x,x.sizeof); }
	void write(wchar x)		{ super.write(&x,x.sizeof); }

	void write(char[] x)	{
		this.write(x.length);
		super.write(x,x.length*char.sizeof);
	}

	void write(wchar[] x)	{
		this.write(x.length);
		super.write(x,x.length*wchar.sizeof);
	}

	void write(dchar[] x)	{
		this.write(x.length);
		super.write(x,x.length*dchar.sizeof);
	}

	void writeLine(char[] x){
		super.write(x~EOL,(x.length+EOL.length)*char.sizeof);
	}
}

class BufferedStream(alias T,uint K) : T
{
	this()
	{
	}

	ulong available()
	{
		ulong avail;

		if (pread <= pwrite) avail = pwrite-pread;
		else {
			avail = buffer.length-pread;
			avail += pwrite;
		}
		return super.available() + avail;
	}

	alias T.read read;

	ulong read(ubyte[] data)
	{
		ulong end,bytes,r;
	
		while (bytes != data.length) {
			if (pwrite >= pread) end = pwrite;
			else end = buffer.length;

			if (end-pread) {
				if (end-pread > data.length-bytes)
					end = pread+data.length-bytes;

				data[bytes..bytes+end-pread] = buffer[pread..end];
				bytes += end-pread;
				pread = end;

				if (pwrite == buffer.length)
					pwrite = 0;
			}
			else {
				if (pread > pwrite) end = pread;
				else end = buffer.length;

				r = super.read(buffer[pwrite..end],end-pwrite);
				if (r == 0) break;
				pwrite += r;

				if (pread == buffer.length)
					pread = 0;
			}

		}

		return bytes;
	}

	alias T.write write;

	ulong write(ubyte[] data)
	{
		return super.write(data,data.length);
	}

private:
	ubyte[K] buffer;
	uint pread;
	uint pwrite;
}

------------HThpJ5CpXXRdSLQVK4P5jO--
Aug 02 2004
next sibling parent "Matthew" <admin stlsoft.dot.dot.dot.dot.org> writes:
"Regan Heath" <regan netwin.co.nz> wrote in message
news:opsb4onie15a2sq9 digitalmars.com...

 Typical stream implementations I have seen use the following heirarchy:

 class Stream {
 }
 class FileStream : Stream {
 }
 class SocketStream : Stream {
 }

 in other words you have to design/derive a class from Stream for each type
 of Stream you want to implement.

 I always thought this was kinda backwards, and limited, in fact after
 Arcane Jill said something along the lines of "anything that reads and
 writes bytes _is_ a stream" I got to thinking surely I can use templates
 to make that statement true.


 The way I have done it is to define some interfaces:

 interface InputStream
 {
 ulong read(void* buffer, ulong length);
 ulong available();
 bool eof();
 }

 interface OutputStream
 {
 ulong write(void* buffer, ulong length);
 }

 and a Stream and BufferedStream template class. Any object that supports
 those interfaces (as does my RawFile class) can then be extended by the
 two template classes to give full stream functionality by some simple
 aliases, example:

 alias Stream!(RawFile) FileStream;
 alias BufferedStream!(FileStream,2048) File;

 I have attached the code, it is far from complete and probably buggy as
 heck as I have not done any testing (or unittests - I know I'm bad) this
 was simply proof of concept.

 Any and all comments are welcome/desired.



They are remarkably similar to two interfaces - IInputStream & IOutputStream -
that I've recently written for a client's
networking project.

I can't include the code, of course. ;)

I agree with the general principle you outline. IStream is one of the elegant
parts of COM. (IStorage one of the hideous
ones.<g>).
Aug 02 2004
prev sibling next sibling parent reply parabolis <parabolis softhome.net> writes:
Regan Heath wrote:


 Typical stream implementations I have seen use the following heirarchy:
 
 class Stream {
 }
 class FileStream : Stream {
 }
 class SocketStream : Stream {
 }
 
 in other words you have to design/derive a class from Stream for each 
 type of Stream you want to implement.
 
 I always thought this was kinda backwards, and limited, in fact after 
 Arcane Jill said something along the lines of "anything that reads and 
 writes bytes _is_ a stream" I got to thinking surely I can use templates 
 to make that statement true.
 
 
 The way I have done it is to define some interfaces:
 
 interface InputStream
 {
     ulong read(void* buffer, ulong length);
     ulong available();
     bool eof();
 }
 
 interface OutputStream
 {
     ulong write(void* buffer, ulong length);
 }
 
 and a Stream and BufferedStream template class. Any object that supports 
 those interfaces (as does my RawFile class) can then be extended by the 
 two template classes to give full stream functionality by some simple 
 aliases, example:
 
 alias Stream!(RawFile) FileStream;
 alias BufferedStream!(FileStream,2048) File;
 
 I have attached the code, it is far from complete and probably buggy as 
 heck as I have not done any testing (or unittests - I know I'm bad) this 
 was simply proof of concept.
 
 Any and all comments are welcome/desired.
 
 Regan



I am not entirely clear on how what you are doing is different 
from the typical stream implementations you have seen.

Look through the thread "OT scanf in Java" for a long post of 
mine towards the top that explains why the Stream 
implementations that you think of as being backwards actually do 
exactly what you want.

To give a hint you should ask yourself how you can combine these 
streams:

(--Stream means InputStream or OutputStream)

File--Stream
Network--Stream
Memory--Stream

Buffered--Stream
Compress--Stream
CRC32 Digest--Stream (eg CRC32 or MD5)

Image--Stream
Video--Stream
Audio--Stream


Sometimes I will want my data to come from a File, other times I 
want it to come from the Network. Sometimes I want a digest of 
the compressed data other times I want a digest of the 
uncompressed data.

These actions are performed in the order I want based on the 
order in which I construct them:

new ImageStream( DigestStream( CompressStream ( FileStream ) ) )

This would read file data, decompresses the data and then 
compute the CRC32 of the uncompressed data and read image data 
from it. However:

new ImageStream( CompressStream ( DigestStream( FileStream ) ) )

This would read file data, compute the CRC32 of the uncompressed 
data and then decompress the data and read image data from it.


I suspect the class you want to implement will either duplicate 
the way Stream libraries work or you will find yourself writing 
seperate classes to handle the two cases above which is bad. 
Just try to write out all the possible combinations that these 9 
classes have already accomplished and you will see what I mean.
Aug 02 2004
parent reply Regan Heath <regan netwin.co.nz> writes:
On Mon, 02 Aug 2004 20:21:28 -0400, parabolis <parabolis softhome.net> 
wrote:

 Regan Heath wrote:


 Typical stream implementations I have seen use the following heirarchy:

 class Stream {
 }
 class FileStream : Stream {
 }
 class SocketStream : Stream {
 }

 in other words you have to design/derive a class from Stream for each 
 type of Stream you want to implement.

 I always thought this was kinda backwards, and limited, in fact after 
 Arcane Jill said something along the lines of "anything that reads and 
 writes bytes _is_ a stream" I got to thinking surely I can use 
 templates to make that statement true.


 The way I have done it is to define some interfaces:

 interface InputStream
 {
     ulong read(void* buffer, ulong length);
     ulong available();
     bool eof();
 }

 interface OutputStream
 {
     ulong write(void* buffer, ulong length);
 }

 and a Stream and BufferedStream template class. Any object that 
 supports those interfaces (as does my RawFile class) can then be 
 extended by the two template classes to give full stream functionality 
 by some simple aliases, example:

 alias Stream!(RawFile) FileStream;
 alias BufferedStream!(FileStream,2048) File;

 I have attached the code, it is far from complete and probably buggy as 
 heck as I have not done any testing (or unittests - I know I'm bad) 
 this was simply proof of concept.

 Any and all comments are welcome/desired.

 Regan



 I am not entirely clear on how what you are doing is different from the 
 typical stream implementations you have seen.



I have written the code to read different types etc once in my Stream 
template, and can apply it to _any_ class at all that implements one read 
method of the form:

   ulong read(void* buffer, ulong length);

the same applies to writing.


So, if you added methods in the forms given in my interfaces to std.socket 
you could say:

alias Stream!(Socket) SocketStream;
alias BufferedStream!(SocketStream,2048) BufferedSocket;

and you have a socket you can instantiate like so:

   SocketStream s = new SocketStream();

which is a buffered stream.


 Look through the thread "OT scanf in Java" for a long post of mine 
 towards the top that explains why the Stream implementations that you 
 think of as being backwards actually do exactly what you want.

 To give a hint you should ask yourself how you can combine these streams:



Combination is the next step, what I have so far is not intended to handle 
combinations.


 (--Stream means InputStream or OutputStream)

 File--Stream
 Network--Stream
 Memory--Stream

 Buffered--Stream
 Compress--Stream
 CRC32 Digest--Stream (eg CRC32 or MD5)

 Image--Stream
 Video--Stream
 Audio--Stream


 Sometimes I will want my data to come from a File, other times I want it 
 to come from the Network. Sometimes I want a digest of the compressed 
 data other times I want a digest of the uncompressed data.

 These actions are performed in the order I want based on the order in 
 which I construct them:

 new ImageStream( DigestStream( CompressStream ( FileStream ) ) )

 This would read file data, decompresses the data and then compute the 
 CRC32 of the uncompressed data and read image data from it. However:

 new ImageStream( CompressStream ( DigestStream( FileStream ) ) )

 This would read file data, compute the CRC32 of the uncompressed data 
 and then decompress the data and read image data from it.



My next step is to implement what I call 'filters' they are like your 
CompressStream etc above, they will link one stream to another and filter 
the data passing through them.


 I suspect the class you want to implement will either duplicate the way 
 Stream libraries work or you will find yourself writing seperate classes 
 to handle the two cases above which is bad. Just try to write out all 
 the possible combinations that these 9 classes have already accomplished 
 and you will see what I mean.



As I said above, combinations are my next step. I may end up handling them 
in the same way as you have shown above but I'd rather not, I dislike 
instantiation in the form:

   new ImageStream( CompressStream ( DigestStream( FileStream ) ) )

Regan.

-- 
Using M2, Opera's revolutionary e-mail client: http://www.opera.com/m2/
Aug 02 2004
parent reply parabolis <parabolis softhome.net> writes:
Regan Heath wrote:


 As I said above, combinations are my next step. I may end up handling 
 them in the same way as you have shown above but I'd rather not, I 
 dislike instantiation in the form:
 
   new ImageStream( CompressStream ( DigestStream( FileStream ) ) )



What will your eq. version of the above be?
Aug 02 2004
parent Regan Heath <regan netwin.co.nz> writes:
On Mon, 02 Aug 2004 21:00:57 -0400, parabolis <parabolis softhome.net> 
wrote:


 Regan Heath wrote:


 As I said above, combinations are my next step. I may end up handling 
 them in the same way as you have shown above but I'd rather not, I 
 dislike instantiation in the form:

   new ImageStream( CompressStream ( DigestStream( FileStream ) ) )



 What will your eq. version of the above be?



I'm not at all sure yet.

Regan.

-- 
Using M2, Opera's revolutionary e-mail client: http://www.opera.com/m2/
Aug 02 2004
prev sibling parent reply Ben Hinkle <bhinkle4 juno.com> writes:
Regan Heath wrote:


 Typical stream implementations I have seen use the following heirarchy:
 
 class Stream {
 }
 class FileStream : Stream {
 }
 class SocketStream : Stream {
 }
 
 in other words you have to design/derive a class from Stream for each type
 of Stream you want to implement.
 
 I always thought this was kinda backwards, and limited, in fact after
 Arcane Jill said something along the lines of "anything that reads and
 writes bytes _is_ a stream" I got to thinking surely I can use templates
 to make that statement true.
 
 
 The way I have done it is to define some interfaces:
 
 interface InputStream
 {
 ulong read(void* buffer, ulong length);
 ulong available();
 bool eof();
 }
 
 interface OutputStream
 {
 ulong write(void* buffer, ulong length);
 }
 
 and a Stream and BufferedStream template class. Any object that supports
 those interfaces (as does my RawFile class) can then be extended by the
 two template classes to give full stream functionality by some simple
 aliases, example:
 
 alias Stream!(RawFile) FileStream;
 alias BufferedStream!(FileStream,2048) File;
 
 I have attached the code, it is far from complete and probably buggy as
 heck as I have not done any testing (or unittests - I know I'm bad) this
 was simply proof of concept.
 
 Any and all comments are welcome/desired.
 
 Regan
 



cool idea(s). I like the "bolt-in" (or it bolt-on?) stuff. One possible
problem, though, is that if read(out int x), for example, is in
Stream!(File) and it call super.read(...) then the
BufferedStream!(FileStream) subclass will not read ints from the buffer
(the super.read goes directly to RawFile.read - not BufferedStream.read). I
guess I'm a little lost about how the buffering works.

At one point there was the suggestion of moving the default implementations
of all the read(out int x) etc from Stream (the original one in
std/stream.d) to a mixin so that instead of subclassing to get the default
implementation you mix it in. I don't know how that approach would change
around your code but it's worth thinking about. 
To be more concrete, here's how that would work:
template DefaultInputStream() {
  void read(out int x) {readExact(...);}
  char[] readLine() {...}
  ...
}
template DefaultOutputStream() {
  void write(int x) {...}
  void writeLine(char[] x) {...}
  ...
}
class Stream : InputStream, OutputStream {
 mixin DefaultInputStream;
 mixin DefaultOutputStream;
 ...
}

Anyhow, keep pluggin'.

-Ben
Aug 02 2004
next sibling parent reply Regan Heath <regan netwin.co.nz> writes:
On Mon, 02 Aug 2004 22:19:19 -0400, Ben Hinkle <bhinkle4 juno.com> wrote:

 Regan Heath wrote:


 Typical stream implementations I have seen use the following heirarchy:

 class Stream {
 }
 class FileStream : Stream {
 }
 class SocketStream : Stream {
 }

 in other words you have to design/derive a class from Stream for each 
 type
 of Stream you want to implement.

 I always thought this was kinda backwards, and limited, in fact after
 Arcane Jill said something along the lines of "anything that reads and
 writes bytes _is_ a stream" I got to thinking surely I can use templates
 to make that statement true.


 The way I have done it is to define some interfaces:

 interface InputStream
 {
 ulong read(void* buffer, ulong length);
 ulong available();
 bool eof();
 }

 interface OutputStream
 {
 ulong write(void* buffer, ulong length);
 }

 and a Stream and BufferedStream template class. Any object that supports
 those interfaces (as does my RawFile class) can then be extended by the
 two template classes to give full stream functionality by some simple
 aliases, example:

 alias Stream!(RawFile) FileStream;
 alias BufferedStream!(FileStream,2048) File;

 I have attached the code, it is far from complete and probably buggy as
 heck as I have not done any testing (or unittests - I know I'm bad) this
 was simply proof of concept.

 Any and all comments are welcome/desired.

 Regan



 cool idea(s). I like the "bolt-in" (or it bolt-on?) stuff.



Yeah, I think they're a great idea.


 One possible
 problem, though, is that if read(out int x), for example, is in
 Stream!(File) and it call super.read(...) then the
 BufferedStream!(FileStream) subclass will not read ints from the buffer
 (the super.read goes directly to RawFile.read - not 
 BufferedStream.read). I
 guess I'm a little lost about how the buffering works.



You're right. :(

I think to fix it I change:

   alias Stream!(RawFile) FileStream;
   alias BufferedStream!(FileStream,2048) File;

to

   alias BufferedStream!(RawFile,2048) File;
   alias Stream!(File) FileStream;

and then change the signatures of the methods in BufferedStream from

   ulong read(ubyte[] data)

to

   ulong read(void* buffer, ulong length)

etc.

This highlights an 'inclusion order' weakness to this design which needs 
to be fixed.


 At one point there was the suggestion of moving the default 
 implementations
 of all the read(out int x) etc from Stream (the original one in
 std/stream.d) to a mixin so that instead of subclassing to get the 
 default
 implementation you mix it in. I don't know how that approach would change
 around your code but it's worth thinking about.



I think this might just be the solution to the inclusion order weakness in 
my current design.
I'm going to turn my Stream template class into a pair of mixins.


 To be more concrete, here's how that would work:
 template DefaultInputStream() {
   void read(out int x) {readExact(...);}
   char[] readLine() {...}
   ...
 }
 template DefaultOutputStream() {
   void write(int x) {...}
   void writeLine(char[] x) {...}
   ...
 }
 class Stream : InputStream, OutputStream {
  mixin DefaultInputStream;
  mixin DefaultOutputStream;
  ...
 }

 Anyhow, keep pluggin'.



Thanks. I'll post version 2 when I have one.

Regan

-- 
Using M2, Opera's revolutionary e-mail client: http://www.opera.com/m2/
Aug 02 2004
parent reply "Vathix" <vathixSpamFix dprogramming.com> writes:
 and then change the signatures of the methods in BufferedStream from

    ulong read(ubyte[] data)

 to

    ulong read(void* buffer, ulong length)

 etc.



Why not use void[] for general read and write?
Aug 02 2004
parent Regan Heath <regan netwin.co.nz> writes:
On Tue, 3 Aug 2004 00:19:15 -0400, Vathix <vathixSpamFix dprogramming.com> 
wrote:

 and then change the signatures of the methods in BufferedStream from

    ulong read(ubyte[] data)

 to

    ulong read(void* buffer, ulong length)

 etc.



 Why not use void[] for general read and write?



So you can go:

bool read(out byte x)	{ if (this.read(&x,x.sizeof) ..etc..
bool read(out ubyte x)	{ if (this.read(&x,x.sizeof) ..etc..
bool read(out short x)	{ if (this.read(&x,x.sizeof) ..etc..

Regan.

-- 
Using M2, Opera's revolutionary e-mail client: http://www.opera.com/m2/
Aug 02 2004
prev sibling parent Sean Kelly <sean f4.ca> writes:
In article <cemsn6$kb9$1 digitaldaemon.com>, Ben Hinkle says...

At one point there was the suggestion of moving the default implementations
of all the read(out int x) etc from Stream (the original one in
std/stream.d) to a mixin so that instead of subclassing to get the default
implementation you mix it in.



This is kind of how the stream implementation I was working on is done.  If I
find the time to finish it I'll post it.

Sean
Aug 02 2004