digitalmars.D - How to implement parallelization well.
- Johan Granberg <lijat.meREM OVEgmail.com> Jul 13 2006
- Johan Granberg <lijat.meREM OVEgmail.com> Jul 15 2006
- Juan Jose Comellas <jcomellas gmail.com> Jul 15 2006
- Johan Granberg <lijat.meREM OVEgmail.com> Jul 15 2006
- Juan Jose Comellas <jcomellas gmail.com> Jul 15 2006
- Johan Granberg <lijat.meREM OVEgmail.com> Jul 16 2006
- Frank Benoit <keinfarbton nospam.xyz> Jul 16 2006
How would one go about to implement this function effectively.
//parallel.d
///executes functions in parallel with the first function executed in the
///calling thread. returns when all threads have finished
void parallelize(void delegate()[] functions)
{
}
//end of file
I have hear that it is expensive to create new threads so would it bee
wise too keep the same ones around?
The function is not responsible for any locking of shared resources so
that part is done.
ps. am I right when I think the above can bee called as
array.parallelize it is not important but it would bee cool :)
If wee had array literals we could do [{a();},{b();},{c();}].parallelize
Jul 13 2006
I managed to get it to the point where it sortof works but then crashes
after about 5 seconds. The crash is always at the same line which
consists of a call to Thread.yield and it appears when the gc is doing a
full collect
//thread 1
#0 0x9002c3d8 in semaphore_wait_trap ()
(gdb) up
#1 0x0001ed54 in _D3std6thread6Thread8pauseAllFZv () at util.d:1
1 module sige.util;
(gdb) up
#2 0x00034510 in _D3gcx3Gcx11fullcollectFPvZk () at util.d:1
1 module sige.util;
(gdb) up
#3 0x00034c8c in _D3gcx3Gcx16fullcollectshellFZk () at util.d:1
1 module sige.util;
//thread 2
Program received signal SIGUSR1, User defined signal 1.
[Switching to process 1614 thread 0x4807]
0x9002c368 in swtch_pri ()
(gdb) up
#1 0x9002c334 in sched_yield ()
(gdb) up
#2 0x00007d94 in _D8parallel10WorkThread4mainFZi (this= 0x503200) at
parallel.d:36
//code
private import std.thread;
private class Signal
{
bool done=false;
}
private class WorkThread:Thread//TODO check if thread safe if not make it
{
Signal signal;
bool free=false;
bool job=false;
void delegate() work;
private int main()
{
while(!free)
if(job)
{
void delegate() dg;
synchronized(this)
{
dg=work;
}
dg();
synchronized(this)
{
work=null;
signal.done=true;
signal=null;
job=false;
}
}
else
yield();//hanged here
return 0;
}
this()
{
super(&main,0);
this.start();
}
Signal execute(void delegate() dg)
{
synchronized(this)
{
work=dg;
job=true;
return signal=new Signal();
}
}
void release()
{
free=true;
}
bool idle()
{
synchronized(this)
{
return !signal;
}
}
}
private class Lock
{
WorkThread[] threads;
private WorkThread expand()
{
synchronized(this)
{
threads.length=threads.length+1;
return threads[$-1]=new WorkThread();
}
}
private WorkThread free_thread()
{
synchronized(this)
{
foreach(t;threads)
if(t.idle())
return t;
}
return expand();
}
}
private class ThreadPool//TODO check so it is thread safe (looks ok. ask?)
{
private static Lock lock;
static this()
{
lock=new Lock();
}
static Signal execute(void delegate() dg)
{
return lock.free_thread().execute(dg);
}
}
private bool running(Signal[] s)
{
foreach(b;s)
if(!b.done)
return true;
return false;
}
///executes functions in parallel with the first function executed in the
///calling thread. returns when all threads have finished
void parallelize(void delegate()[] functions)
{
Signal[] s;
if(!functions)
return;
synchronized
{
if(functions.length>1)
{
s.length=functions.length-1;
foreach(i,f;functions[1..$])
if(f)
s[i]=ThreadPool.execute(f);
}
}
if(functions[0])
functions[0]();
while(running(s))Thread.yield();
}
uint cores()
{
}
Jul 15 2006
The SIGUSR1 signal is the one thrown by D's garbage collector. In order to see the real segfault under GDB you need to execute the following command while inside GDB: handle SIGUSR1 SIGUSR2 nostop noprint Johan Granberg wrote:I managed to get it to the point where it sortof works but then crashes after about 5 seconds. The crash is always at the same line which consists of a call to Thread.yield and it appears when the gc is doing a full collect //thread 1 #0 0x9002c3d8 in semaphore_wait_trap () (gdb) up #1 0x0001ed54 in _D3std6thread6Thread8pauseAllFZv () at util.d:1 1 module sige.util; (gdb) up #2 0x00034510 in _D3gcx3Gcx11fullcollectFPvZk () at util.d:1 1 module sige.util; (gdb) up #3 0x00034c8c in _D3gcx3Gcx16fullcollectshellFZk () at util.d:1 1 module sige.util; //thread 2 Program received signal SIGUSR1, User defined signal 1. [Switching to process 1614 thread 0x4807] 0x9002c368 in swtch_pri () (gdb) up #1 0x9002c334 in sched_yield () (gdb) up #2 0x00007d94 in _D8parallel10WorkThread4mainFZi (this= 0x503200) at parallel.d:36 //code private import std.thread; private class Signal { bool done=false; } private class WorkThread:Thread//TODO check if thread safe if not make it { Signal signal; bool free=false; bool job=false; void delegate() work; private int main() { while(!free) if(job) { void delegate() dg; synchronized(this) { dg=work; } dg(); synchronized(this) { work=null; signal.done=true; signal=null; job=false; } } else yield();//hanged here return 0; } this() { super(&main,0); this.start(); } Signal execute(void delegate() dg) { synchronized(this) { work=dg; job=true; return signal=new Signal(); } } void release() { free=true; } bool idle() { synchronized(this) { return !signal; } } } private class Lock { WorkThread[] threads; private WorkThread expand() { synchronized(this) { threads.length=threads.length+1; return threads[$-1]=new WorkThread(); } } private WorkThread free_thread() { synchronized(this) { foreach(t;threads) if(t.idle()) return t; } return expand(); } } private class ThreadPool//TODO check so it is thread safe (looks ok. ask?) { private static Lock lock; static this() { lock=new Lock(); } static Signal execute(void delegate() dg) { return lock.free_thread().execute(dg); } } private bool running(Signal[] s) { foreach(b;s) if(!b.done) return true; return false; } ///executes functions in parallel with the first function executed in the ///calling thread. returns when all threads have finished void parallelize(void delegate()[] functions) { Signal[] s; if(!functions) return; synchronized { if(functions.length>1) { s.length=functions.length-1; foreach(i,f;functions[1..$]) if(f) s[i]=ThreadPool.execute(f); } } if(functions[0]) functions[0](); while(running(s))Thread.yield(); } uint cores() { }
Jul 15 2006
Juan Jose Comellas wrote:The SIGUSR1 signal is the one thrown by D's garbage collector. In order to see the real segfault under GDB you need to execute the following command while inside GDB: handle SIGUSR1 SIGUSR2 nostop noprint
When should I run that? I ran it after the program crashed to no effect and then I tried before starting the program and when i did that the program did not crash at all (I suppose the second thread crashed). Currently I'm using gdb like this gdb path/to/program r (various commands to find in what state things crashed)
Jul 15 2006
You have to do: gdb path/to/program [inside the gdb console] handle SIGUSR1 SIGUSR2 nostop noprint run Johan Granberg wrote:Juan Jose Comellas wrote:The SIGUSR1 signal is the one thrown by D's garbage collector. In order to see the real segfault under GDB you need to execute the following command while inside GDB: handle SIGUSR1 SIGUSR2 nostop noprint
When should I run that? I ran it after the program crashed to no effect and then I tried before starting the program and when i did that the program did not crash at all (I suppose the second thread crashed). Currently I'm using gdb like this gdb path/to/program r (various commands to find in what state things crashed)
Jul 15 2006
Juan Jose Comellas wrote:You have to do: gdb path/to/program [inside the gdb console] handle SIGUSR1 SIGUSR2 nostop noprint run
I run the above and now the program run without stopping but when I changed the noprint to print, it printed SIGUSR1 and the alternating with SIGUSR2 repeating this with a couple of seconds pause inbetwean.Juan Jose Comellas wrote:The SIGUSR1 signal is the one thrown by D's garbage collector. In order to see the real segfault under GDB
How do I check for the "real segfault" when I have done that Thanks in advance
Jul 16 2006
Frank Benoit <keinfarbton nospam.xyz> How do I check for the "real segfault" when I have done that
When it does not break any more, then there is no segmentation fault. Your program works, congrats :)
Jul 16 2006